"""
Parse a Haka metadata file and print out attribute-filter for it.
Short term manual hack for IdP 2 as Haka stopped releasing attribute-filter file.

Usage help: ./manage.py cleandb -h
"""
from django.core.management.base import BaseCommand
from lxml import etree

from rr.models.attribute import Attribute

def haka_attribute_parser(filename):
    """
    Using CamelCase instead of regular underscore attribute names in element tree.
    """
    parser = etree.XMLParser(
        ns_clean=True, remove_comments=True, remove_blank_text=True, resolve_entities=False, no_network=True
    )
    tree = etree.parse(filename, parser)
    root = tree.getroot()
    attribute_filter_policy_group = etree.Element(
        "AttributeFilterPolicyGroup", id="urn:mace:funet.fi:haka", nsmap={"xmlns": "urn:mace:shibboleth:2.0:afp"}
    )
    attribute_filter_policy_group.attrib["{urn:mace:shibboleth:2.0:afp}basic"] = "urn:mace:shibboleth:2.0:afp:mf:basic"
    attribute_filter_policy_group.attrib["{urn:mace:shibboleth:2.0:afp}saml"] = "urn:mace:shibboleth:2.0:afp:mf:saml"
    attribute_filter_policy_group.attrib["{http://www.w3.org/2001/XMLSchema-instance}schemaLocation"] = (
        "urn:mace:shibboleth:2.0:afp classpath:/schema/shibboleth-2.0-afp.xsd "
        "urn:mace:shibboleth:2.0:afp:mf:basic "
        "classpath:/schema/shibboleth-2.0-afp-mf-basic.xsd "
        "urn:mace:shibboleth:2.0:afp:mf:saml "
        "classpath:/schema/shibboleth-2.0-afp-mf-saml.xsd"
    )
    for a in root:
        entity_id = a.get("entityID")
        if entity_id:
            for b in a:
                if etree.QName(b.tag).localname == "SPSSODescriptor":
                    attributes = []
                    for c in b:
                        if etree.QName(c.tag).localname == "AttributeConsumingService":
                            for d in c:
                                if etree.QName(d.tag).localname == "RequestedAttribute":
                                    friendlyname = d.get("FriendlyName")
                                    name = d.get("Name")
                                    if friendlyname:
                                        attribute = Attribute.objects.filter(name=name).first()
                                        if not attribute:
                                            attribute = Attribute.objects.filter(friendlyname=friendlyname).first()
                                        if attribute:
                                            attributes.append(attribute)
                                        else:
                                            print(
                                                "Could not add attribute "
                                                + friendlyname
                                                + ", "
                                                + name
                                                + " for "
                                                + entity_id
                                            )
                    if attributes:
                        attribute_filter_policy = etree.SubElement(
                            attribute_filter_policy_group, "AttributeFilterPolicy", id="haka-default-" + entity_id
                        )
                        policy_requirement_rule = etree.SubElement(
                            attribute_filter_policy, "PolicyRequirementRule", value=entity_id
                        )
                        policy_requirement_rule.attrib[
                            "{http://www.w3.org/2001/XMLSchema-instance}type"
                        ] = "basic:AttributeRequesterString"
                        for attribute in attributes:
                            attribute_rule = etree.SubElement(
                                attribute_filter_policy, "AttributeRule", attributeID=attribute.attributeid
                            )
                            permit_value_rule = etree.SubElement(attribute_rule, "PermitValueRule")
                            permit_value_rule.attrib["{http://www.w3.org/2001/XMLSchema-instance}type"] = "basic:ANY"
    return etree.tostring(attribute_filter_policy_group, pretty_print=True, encoding="UTF-8")

class Command(BaseCommand):
    def add_arguments(self, parser):
        parser.add_argument("-i", type=str, action="store", dest="input", help="Metadata input file name")
        parser.add_argument("-o", type=str, action="store", dest="output", help="Attribute-filter output file name")

    def handle(self, *args, **options):
        metadata_input = options["input"]
        attribute_output = options["output"]
        if metadata_input and attribute_output:
            data = haka_attribute_parser(metadata_input)
            with open(attribute_output, "wb") as f:
                f.write('<?xml version="1.0" encoding="UTF-8"?>\n'.encode("utf-8"))
                # Hack for correcting namespace definition by removing prefix.
                f.write(data.replace(b"xmlns:xmlns", b"xmlns"))
        else:
            self.stdout.write("Please give both input and output files")


import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt

cor = []
cen = []
equ = []
for i in range(325):
    print("Counting %d.csv ..." % i)
    cor += [len(open("Correct/%d.csv" % i, 'r').readlines()) / 1000]
    cen += [len(open("Centralized/%d.csv" % i, 'r').readlines()) / 1000]
    equ += [len(open("Equalized/%d.csv" % i, 'r').readlines()) / 1000]

print("Plotting...")
figure = plt.figure(figsize=(1800/300, 600/300), dpi=300)
plt.subplot(131)
plt.plot(range(325), cor, 'r', label="Correct", lw=0.3)
plt.xlabel("Records(K)")
plt.legend(fontsize="xx-small", loc="upper left")
plt.subplot(132)
plt.plot(range(325), cen, 'b', label="Centralized", lw=0.3)
plt.xlabel("Records(K)")
plt.legend(fontsize="xx-small", loc="upper left")
plt.subplot(133)
plt.plot(range(325), equ, 'g', label="Equalized", lw=0.3)
plt.xlabel("Records(K)")
plt.legend(fontsize="xx-small", loc="upper left")
plt.savefig("Count.png")
print("Done.")


import pickle
import asyncio
import json
import pprint

from indy import pool, ledger, wallet, did, anoncreds
from indy.error import ErrorCode, IndyError
from write_did_functions import print_log

async def schema_request(pool_,submitter):
    # 9.
    print_log('\n9. Build the SCHEMA request to add new schema to the ledger as a Steward\n')

    # Define Schema:
    schema = {
        'data': {
            'name': 'gvt',
            'version': '1.0',
            'ver': '1.0',
            'attributes': ["age", "sex", "height", "name"]
        }
    }

    # This function call creates the schema and assumes the issuer of credentials using the schema to be the
    # submitter of the schema request:

    schema['id'],schema['json'] = await anoncreds.issuer_create_schema(issuer_did=submitter['did'],
                                                                        name=schema['data']['name'],
                                                                        version=schema['data']['version'],
                                                                        attrs=json.dumps(schema['data']['attributes']))

    print_log('Schema data: ')
    pprint.pprint(schema['data'])
    print_log('Schema json: ')
    pprint.pprint(json.loads(schema['json']))
    print_log('Schema: ')
    pprint.pprint(schema)

    # Create schema for an issuer: return issuer_schema_id, issuer_schema_json:

    schema_request = await ledger.build_schema_request(submitter['did'],schema['json'])
    print_log('Schema request: ') # form of a json string
    pprint.pprint(json.loads(schema_request))

    # 10.
    print_log('\n10. Sending the SCHEMA request to the ledger\n')

    schema_response = await ledger.sign_and_submit_request(pool_handle=pool_['handle'],
                                                            wallet_handle=submitter['wallet'],
                                                            submitter_did=submitter['did'],
                                                            request_json=schema_request)

    print_log('Schema response:')
    pprint.pprint(json.loads(schema_response))
    schema_response_dict = json.loads(schema_response)

    if schema_response_dict['op'] == 'REJECT':
        print_log('\nSending GET SCHEMA request to the ledger for existing Version Number\n')
        schema_response = await get_schema_request(pool_,submitter,schema)

    print_log('Schema response:')
    pprint.pprint(json.loads(schema_response))

    return(schema)

async def get_schema_request(pool_,submitter,schema):

    schema_request = await ledger.build_get_schema_request(submitter['did'],schema['id'])
    schema_response = await ledger.sign_and_submit_request(pool_handle=pool_['handle'],
                                                            wallet_handle=submitter['wallet'],
                                                            submitter_did=submitter['did'],
                                                            request_json=schema_request)
    return(schema_response)

async def credential_definition(pool_,issuer,schema):

    # Define a credential definition:
    # Cred['def'] : - id
    #               - json
    #               - tag
    #               - type
    #               - config

    # 11.
    print_log('\n11. Creating and storing CRED DEFINITION using anoncreds as Trust Anchor, for the given Schema\n')

    cred_def = {'Issuer':issuer['name']}
    cred_def['tag'] = 'cred_def_tag'
    cred_def['type'] = 'CL'
    cred_def['config'] = json.dumps({"support_revocation": False})

    (cred_def['id'], cred_def['json']) = await anoncreds.issuer_create_and_store_credential_def(wallet_handle=issuer['wallet'],
                                                                                issuer_did=issuer['did'],
                                                                                schema_json=schema['json'],
                                                                                tag=cred_def['tag'],
                                                                                signature_type=cred_def['type'],
                                                                                config_json=cred_def['config'])
    print_log('Credential definition: ')
    pprint.pprint(json.loads(cred_def['json']))

    return(cred_def)


#!/usr/env python
"""
.. module:: convert_kids450photometry_h5py
:synopsis: Script to get the KIDS 450 photometry from the shear catalog and save as a h5 file.
.. moduleauthor:: Maria Elidaiana <mariaeli@brandeis.edu>
"""

import numpy as np
#from astropy.io import fits
from astropy.table import Table, vstack
import h5py as h5

kidsdir = '/global/projecta/projectdirs/lsst/groups/WL/projects/wl-txpipe-hack/KIDS/KiDS450_DR3/'

def getdata(filename):
    kids_shear = Table.read(kidsdir + filename, memmap=True)
    return kids_shear

kidsfiles = ['KiDS_DR3.1_G9_ugri_shear.fits', 'KiDS_DR3.1_G12_ugri_shear.fits',
             'KiDS_DR3.1_G15_ugri_shear.fits', 'KiDS_DR3.1_G23_ugri_shear.fits',
             'KiDS_DR3.1_GS_ugri_shear.fits']
tables = []
for i in range(len(kidsfiles)):
    d = getdata(kidsfiles[i])
    tables.append(d)

#Joining the tables
kids_shearall = vstack(tables)

#Sorting by ID
kids_shearall.sort('ID')

#Getting the photometry columns
tilename  = kids_shearall['KIDS_TILE']
dec       = kids_shearall['DECJ2000']
g_mag     = kids_shearall['MAG_g']
g_mag_err = kids_shearall['MAGERR_g']
i_mag     = kids_shearall['MAG_i']
i_mag_err = kids_shearall['MAGERR_i']
objectId  = kids_shearall['ID']
r_mag     = kids_shearall['MAG_r']
r_mag_err = kids_shearall['MAGERR_r']
ra        = kids_shearall['RAJ2000']
snr_g     = 1.086/g_mag_err
snr_i     = 1.086/i_mag_err
snr_r     = 1.086/r_mag_err
u_mag     = kids_shearall['MAG_u']
u_mag_err = kids_shearall['MAGERR_u']
snr_u     = 1.086/u_mag_err
y_mag     = kids_shearall['MAG_i']    #Placeholder
y_mag_err = kids_shearall['MAGERR_i'] #Placeholder
snr_y     = 1.086/y_mag_err           #Placeholder
z_mag     = kids_shearall['MAG_i']    #Placeholder
z_mag_err = kids_shearall['MAGERR_i'] #Placeholder
snr_z     = 1.086/z_mag_err           #Placeholder

#Dealing with unicode, string needs to be S12
tilename = np.array([a.encode('utf8') for a in tilename])
objectId = np.array([a.encode('utf8') for a in objectId])

data   = [tilename, dec, g_mag, g_mag_err, i_mag, i_mag_err, objectId, r_mag, r_mag_err, ra, snr_g, snr_i, snr_r, u_mag, u_mag_err, snr_u, y_mag, y_mag_err, snr_y, z_mag, z_mag_err, snr_z]
dnames = ['tilename', 'dec', 'g_mag', 'g_mag_err', 'i_mag', 'i_mag_err', 'objectId', 'r_mag', 'r_mag_err', 'ra', 'snr_g', 'snr_i', 'snr_r', 'u_mag', 'u_mag_err', 'snr_u', 'y_mag', 'y_mag_err', 'snr_y', 'z_mag', 'z_mag_err', 'snr_z']

outputdir = '/global/cscratch1/sd/elp25/txpipe-reanalysis/data/kids/'
#Saving the h5 file...
f = h5.File(outputdir + 'photometry_catalog_kids450.h5', 'w')
g = f.create_group('photometry')
for i in range(len(data)):
    g.create_dataset(dnames[i], data=data[i], dtype=data[i].dtype)
f.close()


from __future__ import absolute_import
from __future__ import unicode_literals
import gevent
import os
import re
import subprocess

from django.test import SimpleTestCase

class TestRequireJS(SimpleTestCase):

    @classmethod
    def _run_jobs(cls, files, processor):
        jobs = []
        for filename in files:
            jobs.append(gevent.spawn(processor, filename))
        gevent.joinall(jobs)

    @classmethod
    def setUpClass(cls):
        super(TestRequireJS, cls).setUpClass()
        prefix = os.path.join(os.getcwd(), 'corehq')

        proc = subprocess.Popen(["find", prefix, "-name", "*.js"], stdout=subprocess.PIPE)
        (out, err) = proc.communicate()
        cls.js_files = [f for f in out.split("\n") if f
                    and not re.search(r'/_design/', f)
                    and not re.search(r'couchapps', f)
                    and not re.search(r'/vellum/', f)]

        cls.hqdefine_files = []
        cls.requirejs_files = []

        def _categorize_file(filename):
            proc = subprocess.Popen(["grep", "^\s*hqDefine", filename], stdout=subprocess.PIPE)
            (out, err) = proc.communicate()
            if out:
                cls.hqdefine_files.append(filename)
                proc = subprocess.Popen(["grep", "hqDefine.*,.*\[", filename], stdout=subprocess.PIPE)
                (out, err) = proc.communicate()
                if out:
                    cls.requirejs_files.append(filename)

        cls._run_jobs(cls.js_files, _categorize_file)

    def test_files_match_modules(self):
        errors = []

        def _test_file(filename):
            proc = subprocess.Popen(["grep", "hqDefine", filename], stdout=subprocess.PIPE)
            (out, err) = proc.communicate()
            for line in out.split("\n"):
                match = re.search(r'^\s*hqDefine\([\'"]([^\'"]*)[\'"]', line)
                if match:
                    module = match.group(1)
                    if not filename.endswith(module + ".js"):
                        errors.append("Module {} defined in file {}".format(module, filename))

        self._run_jobs(self.hqdefine_files, _test_file)

        if errors:
            self.fail("Mismatched JS file/modules: \n{}".format("\n".join(errors)))

    def test_requirejs_disallows_hqimport(self):
        errors = []

        # Special cases:
        #   Ignore standard_hq_report.js until we migrate UCRs and reports
        #   knockout_bindings should be broken up, in the meantime, ignore
        test_files = [f for f in self.requirejs_files
                      if not f.endswith("reports/js/standard_hq_report.js")
                      and not f.endswith("hqwebapp/js/knockout_bindings.ko.js")]

        def _test_file(filename):
            proc = subprocess.Popen(["grep", "hqImport", filename], stdout=subprocess.PIPE)
            (out, err) = proc.communicate()
            for line in out.split("\n"):
                if line:
                    match = re.search(r'hqImport\([\'"]([^\'"]*)[\'"]', line)
                    if match:
                        errors.append("{} imported in {}".format(match.group(1), filename))
                    else:
                        errors.append("hqImport found in {}: {}".format(filename, line.strip()))

        self._run_jobs(test_files, _test_file)

        if errors:
            self.fail("hqImport used in RequireJS modules: \n{}".format("\n".join(errors)))


from __future__ import print_function
import gc, sys #, os
#from copy import deepcopy
from warnings import warn
from time import time
import numpy as np
from scipy.sparse import coo_matrix, csr_matrix, csc_matrix

from ._LowLevelAssembly_ import _LowLevelAssembly_ #, _LowLevelAssemblyExplicit_, _LowLevelAssemblyLaplacian_
#from ._LowLevelAssembly_ import _LowLevelAssembly_Par_, _LowLevelAssemblyExplicit_Par_

from .SparseAssemblyNative import SparseAssemblyNative, SparseAssemblyNativeCSR_RecomputeDataIndex #, SparseAssemblyNativeCSR
from .RHSAssemblyNative import RHSAssemblyNative
#from .ComputeSparsityPattern import ComputeSparsityPattern

# PARALLEL PROCESSING ROUTINES
#import multiprocessing
#import Florence.ParallelProcessing.parmap as parmap

__all__ = ['Assemble', 'AssembleRobinForces','AssembleForces']

#---------------------------------------------------------------------------------------------------#
#------------------------ ASSEMBLY ROUTINE FOR INTERNAL TRACTION FORCES ----------------------------#
#---------------------------------------------------------------------------------------------------#
def Assemble(fem_solver, function_spaces, formulation, mesh, materials, boundary_condition, Eulerx):

    if fem_solver.has_low_level_dispatcher:
        return LowLevelAssembly(fem_solver, function_spaces, formulation, mesh, materials,
                boundary_condition, Eulerx)
    else:
        if mesh.nelem <= 600000:
            return AssemblySmall(fem_solver, function_spaces, formulation, mesh, materials,
                    boundary_condition, Eulerx)
        elif mesh.nelem > 600000:
            print("Larger than memory system. Dask on disk parallel assembly is turned on")
            return OutofCoreAssembly(fem_solver, function_spaces[0], formulation, mesh, materials,
                    boundary_condition, Eulerx)

def LowLevelAssembly(fem_solver, function_spaces, formulation, mesh, materials, boundary_condition,
        Eulerx):

    t_assembly = time()

    for imat in range(len(materials)):
        if not materials[imat].has_low_level_dispatcher:
            raise RuntimeError("Cannot dispatch to low level module since material {} does not support it".format(type(material).__name__))

    # HACK TO DISPATCH TO EFFICIENT MASS MATRIX COMUTATION
    ll_failed = False
    M = []
    if fem_solver.analysis_type != "static" and fem_solver.is_mass_computed is False:
        try:
            t_mass_assembly = time()
            from Kuru.VariationalPrinciple._MassIntegrand_ import __TotalConstantMassIntegrand__
            if fem_solver.recompute_sparsity_pattern:
                M, I_mass, J_mass, V_mass = __TotalConstantMassIntegrand__(mesh, function_spaces[0],
                        formulation, fem_solver.mass_type)
                if fem_solver.mass_type == "consistent":
                    M = csr_matrix((V_mass,(I_mass,J_mass)),
                            shape=((formulation.nvar*mesh.points.shape[0],
                            formulation.nvar*mesh.points.shape[0])),dtype=np.float64)
            else:
                M, V_mass = __TotalConstantMassIntegrand__(mesh, function_spaces[0],
                    formulation, fem_solver.mass_type, fem_solver.recompute_sparsity_pattern,
                    fem_solver.squeeze_sparsity_pattern, fem_solver.indices, fem_solver.indptr,
                    fem_solver.data_global_indices, fem_solver.data_local_indices)
                if fem_solver.mass_type == "consistent":
                    M = csr_matrix((V_mass,fem_solver.indices,fem_solver.indptr),
                            shape=((formulation.nvar*mesh.points.shape[0],
                        formulation.nvar*mesh.points.shape[0])),dtype=np.float64)
            if M is not None:
                fem_solver.is_mass_computed = True
            t_mass_assembly = time() - t_mass_assembly
            print("Assembled mass matrix. Time elapsed was {} seconds".format(t_mass_assembly))
        except ImportError:
            # CONTINUE DOWN
            warn("Low level mass assembly not available. Falling back to python version")
            ll_failed = True

    if fem_solver.parallel:
        stiffness, T, mass = ImplicitParallelLauncher(fem_solver, function_spaces[0], formulation,
                mesh, materials, Eulerx)
    else:
        stiffness, T, mass = _LowLevelAssembly_(fem_solver, function_spaces[0], formulation, mesh,
                materials, Eulerx)

    # SET FLAG AGAIN - NECESSARY
    if ll_failed:
        if mass is not None:
            fem_solver.is_mass_computed = True
    else:
        mass = M

    fem_solver.assembly_time = time() - t_assembly

    return stiffness, T[:,None], mass

def AssemblySmall(fem_solver, function_spaces, formulation, mesh, materials, boundary_condition, Eulerx):

    t_assembly = time()

    # GET MESH DETAILS
    C = mesh.InferPolynomialDegree() - 1
    nvar = formulation.nvar
    ndim = formulation.ndim
    nelem = mesh.nelem
    nodeperelem = mesh.elements.shape[1]
    ndof = nodeperelem*nvar
    local_capacity = ndof*ndof

    if fem_solver.recompute_sparsity_pattern is False:
        indices, indptr = fem_solver.indices, fem_solver.indptr
        if fem_solver.squeeze_sparsity_pattern is False:
            data_global_indices = fem_solver.data_global_indices
            data_local_indices = fem_solver.data_local_indices

    if fem_solver.recompute_sparsity_pattern:
        # ALLOCATE VECTORS FOR SPARSE ASSEMBLY OF STIFFNESS MATRIX - CHANGE TYPES TO INT64 FOR DoF > 1e09
        I_stiffness=np.zeros(int((nvar*nodeperelem)**2*nelem),dtype=np.int32)
        J_stiffness=np.zeros(int((nvar*nodeperelem)**2*nelem),dtype=np.int32)
        V_stiffness=np.zeros(int((nvar*nodeperelem)**2*nelem),dtype=np.float64)

        I_mass=[]; J_mass=[]; V_mass=[]
        if fem_solver.analysis_type !='static' and fem_solver.is_mass_computed is False:
            # ALLOCATE VECTORS FOR SPARSE ASSEMBLY OF MASS MATRIX - CHANGE TYPES TO INT64 FOR DoF > 1e09
            I_mass=np.zeros(int((nvar*nodeperelem)**2*nelem),dtype=np.int32)
            J_mass=np.zeros(int((nvar*nodeperelem)**2*nelem),dtype=np.int32)
            V_mass=np.zeros(int((nvar*nodeperelem)**2*nelem),dtype=np.float64)
    else:
        V_stiffness=np.zeros(indices.shape[0],dtype=np.float64)
        if fem_solver.analysis_type !='static' and fem_solver.is_mass_computed is False:
            V_mass=np.zeros(indices.shape[0],dtype=np.float64)

    T = np.zeros((mesh.points.shape[0]*nvar,1),np.float64)

    mass = []

    if fem_solver.parallel:
        # COMPUATE ALL LOCAL ELEMENTAL MATRICES (STIFFNESS, MASS, INTERNAL & EXTERNAL TRACTION FORCES)
        ParallelTuple = parmap.map(formulation,np.arange(0,nelem,dtype=np.int32),
            function_spaces[0], mesh, materials, fem_solver, Eulerx, processes= int(multiprocessing.cpu_count()/2))

    for iset in range(len(materials)):
        material = materials[iset]
        for elem in material.element_set:

            if fem_solver.parallel:
                # UNPACK PARALLEL TUPLE VALUES
                I_stiff_elem = ParallelTuple[elem][0]; J_stiff_elem = ParallelTuple[elem][1]; V_stiff_elem = ParallelTuple[elem][2]
                t = ParallelTuple[elem][3]; f = ParallelTuple[elem][4]
                I_mass_elem = ParallelTuple[elem][5]; J_mass_elem = ParallelTuple[elem][6]; V_mass_elem = ParallelTuple[elem][6]

            else:
                # COMPUATE ALL LOCAL ELEMENTAL MATRICES (STIFFNESS, MASS, INTERNAL TRACTION FORCES )
                I_stiff_elem, J_stiff_elem, V_stiff_elem, t, \
                I_mass_elem, J_mass_elem, V_mass_elem = formulation.GetElementalMatrices(elem,
                    function_spaces[0], mesh, material, fem_solver, Eulerx)

            if fem_solver.recompute_sparsity_pattern:
                # SPARSE ASSEMBLY - STIFFNESS MATRIX
                SparseAssemblyNative(I_stiff_elem,J_stiff_elem,V_stiff_elem,I_stiffness,J_stiffness,V_stiffness,
                    elem,nvar,nodeperelem,mesh.elements)

                if fem_solver.analysis_type != 'static' and fem_solver.is_mass_computed==False:
                    # SPARSE ASSEMBLY - MASS MATRIX
                    SparseAssemblyNative(I_mass_elem,J_mass_elem,V_mass_elem,I_mass,J_mass,V_mass,
                        elem,nvar,nodeperelem,mesh.elements)

            else:
                if fem_solver.squeeze_sparsity_pattern:
                    # SPARSE ASSEMBLY - STIFFNESS MATRIX
                    SparseAssemblyNativeCSR_RecomputeDataIndex(mesh,V_stiff_elem,indices,indptr,V_stiffness,elem,nvar)

                    if fem_solver.analysis_type != 'static' and fem_solver.is_mass_computed==False:
                        # SPARSE ASSEMBLY - MASS MATRIX
                        SparseAssemblyNativeCSR_RecomputeDataIndex(mesh,V_mass_elem,indices,indptr,V_mass,elem,nvar)
                else:
                    # SPARSE ASSEMBLY - STIFFNESS MATRIX
                    V_stiffness[data_global_indices[elem*local_capacity:(elem+1)*local_capacity]] \
                        += V_stiff_elem[data_local_indices[elem*local_capacity:(elem+1)*local_capacity]]

                    if fem_solver.analysis_type != 'static' and fem_solver.is_mass_computed==False:
                        # SPARSE ASSEMBLY - MASS MATRIX
                        V_mass[data_global_indices[elem*local_capacity:(elem+1)*local_capacity]] \
                        += V_mass_elem[data_local_indices[elem*local_capacity:(elem+1)*local_capacity]]

            # INTERNAL TRACTION FORCE ASSEMBLY
            # for iterator in range(0,nvar):
                # T[mesh.elements[elem,:]*nvar+iterator,0]+=t[iterator::nvar,0]
            RHSAssemblyNative(T,t,elem,nvar,nodeperelem,mesh.elements)

    if fem_solver.parallel:
        del ParallelTuple
        gc.collect()

    if fem_solver.recompute_sparsity_pattern:
        stiffness = coo_matrix((V_stiffness,(I_stiffness,J_stiffness)),
            shape=((nvar*mesh.points.shape[0],nvar*mesh.points.shape[0])),dtype=np.float64).tocsr()

        # GET STORAGE/MEMORY DETAILS
        fem_solver.spmat = stiffness.data.nbytes/1024./1024.
        fem_solver.ijv = (I_stiffness.nbytes + J_stiffness.nbytes + V_stiffness.nbytes)/1024./1024.

        del I_stiffness, J_stiffness, V_stiffness
        gc.collect()

        if fem_solver.analysis_type != 'static' and fem_solver.is_mass_computed==False:
            mass = csr_matrix((V_mass,(I_mass,J_mass)),shape=((nvar*mesh.points.shape[0],
                nvar*mesh.points.shape[0])),dtype=np.float64)
            fem_solver.is_mass_computed = True

    else:
        stiffness = csr_matrix((V_stiffness,indices,indptr),
            shape=((nvar*mesh.points.shape[0],nvar*mesh.points.shape[0])))

        # GET STORAGE/MEMORY DETAILS
        fem_solver.spmat = stiffness.data.nbytes/1024./1024.
        fem_solver.ijv = (indptr.nbytes + indices.nbytes + V_stiffness.nbytes)/1024./1024.

        if fem_solver.analysis_type != 'static' and fem_solver.is_mass_computed==False:
            mass = csr_matrix((V_mass,indices,indptr),
                shape=((nvar*mesh.points.shape[0],nvar*mesh.points.shape[0])))
            fem_solver.is_mass_computed = True

    fem_solver.assembly_time = time() - t_assembly
    return stiffness, T, mass

#----------------------------------------------------------------------------------------------------------------#
#------------------------------- ASSEMBLY ROUTINE FOR EXTERNAL TRACTION FORCES ----------------------------------#
#----------------------------------------------------------------------------------------------------------------#

#------------------------------- ASSEMBLY ROUTINE FOR EXTERNAL PRESSURE FORCES ----------------------------------#
def AssembleRobinForces(boundary_condition, mesh, material, function_spaces, fem_solver, Eulerx, type_load):
    """Compute/assemble traction (follower)"""

    ndim = mesh.InferSpatialDimension()
    nvar = material.nvar

    if type_load == 'pressure':
        if boundary_condition.pressure_flags.shape[0] == mesh.points.shape[0]:
            #boundary_condition.robin_data_applied_at = "node"
            raise ValueError("Robin boundary forces (pressure) applied at nodes")
    elif type_load == 'spring':
        if boundary_condition.spring_flags.shape[0] == mesh.points.shape[0]:
            #boundary_condition.robin_data_applied_at = "node"
            raise ValueError("Robin boundary forces (spring) applied at nodes")
    elif type_load == 'connector':
        if ndim == 2:
            if boundary_condition.connector_elements.shape[1] != 2*mesh.edges.shape[1]:
                raise ValueError("Robin boundary connector should be compose by two boundary elements")
        else:
            if boundary_condition.connector_elements.shape[1] != 2*mesh.faces.shape[1]:
                raise ValueError("Robin boundary connector should be compose by two boundary elements")
    else:
        raise ValueError("Load {} not unserstood. Just spring or pressure.".format(type_load))

    if not isinstance(function_spaces,tuple):
        raise ValueError("Boundary functional spaces not available for computing pressure stiffness")
    else:
        # CHECK IF A FUNCTION SPACE FOR BOUNDARY EXISTS - SAFEGAURDS AGAINST FORMULATIONS THAT DO NO PROVIDE ONE
        has_boundary_spaces = False
        for fs in function_spaces:
            if ndim == 3 and fs.ndim == 2:
                has_boundary_spaces = True
                break
            elif ndim == 2 and fs.ndim == 1:
                has_boundary_spaces = True
                break
        if not has_boundary_spaces:
            from Kuru import QuadratureRule, FunctionSpace
            # COMPUTE BOUNDARY FUNCTIONAL SPACES
            p = mesh.InferPolynomialDegree()
            bquadrature = QuadratureRule(optimal=3, norder=2*p+1,
                mesh_type=mesh.boundary_element_type, is_flattened=False)
            bfunction_space = FunctionSpace(mesh.CreateDummyLowerDimensionalMesh(),
                bquadrature, p=p, equally_spaced=mesh.IsEquallySpaced, use_optimal_quadrature=False)
            function_spaces = (function_spaces[0],bfunction_space)

    if type_load == 'pressure':
        from .RobinForces import StaticPressureForces
        if boundary_condition.analysis_type == "static":
            if fem_solver.recompute_sparsity_pattern:
                I_robin, J_robin, V_robin, F_robin = StaticPressureForces(boundary_condition,
                    mesh, material, function_spaces[-1], fem_solver, Eulerx)
                K_robin = coo_matrix((V_robin,(I_robin,J_robin)),
                    shape=((nvar*mesh.points.shape[0],nvar*mesh.points.shape[0])),dtype=np.float64).tocsr()
            else:
                V_robin, F_robin = StaticPressureForces(boundary_condition, mesh,
                    material, function_spaces[-1], fem_solver, Eulerx)
                K_robin = csr_matrix((V_robin,fem_solver.indices,fem_solver.indptr),
                    shape=((nvar*mesh.points.shape[0],nvar*mesh.points.shape[0])))

        elif boundary_condition.analysis_type == "dynamic":
            raise ValueError("Not implemented yet")

        return K_robin, F_robin

    if type_load == 'spring':
        from .RobinForces import StaticSpringForces
        if boundary_condition.analysis_type == "static":
            if fem_solver.recompute_sparsity_pattern:
                I_robin, J_robin, V_robin, F_robin = StaticSpringForces(boundary_condition,
                    mesh, material, function_spaces[-1], fem_solver, Eulerx)
                K_robin = coo_matrix((V_robin,(I_robin,J_robin)),
                    shape=((nvar*mesh.points.shape[0],nvar*mesh.points.shape[0])),dtype=np.float64).tocsr()
            else:
                V_robin, F_robin = StaticSpringForces(boundary_condition, mesh,
                    material, function_spaces[-1], fem_solver, Eulerx)
                K_robin = csr_matrix((V_robin,fem_solver.indices,fem_solver.indptr),
                    shape=((nvar*mesh.points.shape[0],nvar*mesh.points.shape[0])))

        elif boundary_condition.analysis_type == "dynamic":
            raise ValueError("Not implemented yet")

        return K_robin, F_robin

    if type_load == 'connector':
        from .RobinForces import StaticConnectorForces
        if boundary_condition.analysis_type == "static":
            if fem_solver.recompute_sparsity_pattern:
                I_robin, J_robin, V_robin, F_robin = StaticConnectorForces(boundary_condition,
                    mesh, material, function_spaces[-1], fem_solver, Eulerx)
                K_robin = coo_matrix((V_robin,(I_robin,J_robin)),
                    shape=((nvar*mesh.points.shape[0],nvar*mesh.points.shape[0])),dtype=np.float64).tocsr()
            else:
                V_robin, F_robin = StaticConnectorForces(boundary_condition, mesh,
                    material, function_spaces[-1], fem_solver, Eulerx)
                K_robin = csr_matrix((V_robin,fem_solver.indices,fem_solver.indptr),
                    shape=((nvar*mesh.points.shape[0],nvar*mesh.points.shape[0])))

        elif boundary_condition.analysis_type == "dynamic":
            raise ValueError("Not implemented yet")

        return K_robin, F_robin

#------------------------------- ASSEMBLY ROUTINE FOR EXTERNAL TRACTION FORCES ----------------------------------#

def AssembleForces(boundary_condition, mesh, materials, function_spaces,
        compute_traction_forces=True, compute_body_forces=False):

    Ft = np.zeros((mesh.points.shape[0]*materials[0].nvar,1))
    Fb = np.zeros((mesh.points.shape[0]*materials[0].nvar,1))

    if compute_body_forces:
        Fb = AssembleBodyForces(boundary_condition, mesh, materials, function_spaces[0])
    if compute_traction_forces:
        Ft = AssembleExternalTractionForces(boundary_condition, mesh, materials[0], function_spaces[-1])

    return Ft + Fb

def AssembleExternalTractionForces(boundary_condition, mesh, material, function_space):

    nvar = material.nvar
    ndim = material.ndim
    ngauss = function_space.AllGauss.shape[0]

    if ndim == 2:
        faces = mesh.edges
        nodeperelem = mesh.edges.shape[1]
    else:
        faces = mesh.faces
        nodeperelem = mesh.faces.shape[1]

    if boundary_condition.is_applied_neumann_shape_functions_computed is False:
        N = np.zeros((nodeperelem*nvar,nvar,ngauss))
        for i in range(nvar):
            N[i::nvar,i,:] = function_space.Bases
        boundary_condition.__Nt__ = N
        boundary_condition.is_applied_neumann_shape_functions_computed = True
    else:
        N = boundary_condition.__Nt__

    F = np.zeros((mesh.points.shape[0]*nvar,1))
    for face in range(faces.shape[0]):
        if boundary_condition.neumann_flags[face] == True:
            ElemTraction = boundary_condition.applied_neumann[face,:]
            external_traction = np.einsum("ijk,j,k->ik",N,ElemTraction,function_space.AllGauss[:,0]).sum(axis=1)
            RHSAssemblyNative(F,np.ascontiguousarray(external_traction[:,None]),face,nvar,nodeperelem,faces)

    return F


# https://www.codewars.com/kata/56882731514ec3ec3d000009/train/python

COLUMNS, ROWS = 'ABCDEFG', range(6)
LINES = [{(COLUMNS[i+k], ROWS[j]) for k in range(4)}
           for i in range(len(COLUMNS) - 3) for j in range(len(ROWS))] \
        + [{(COLUMNS[i], ROWS[j+k]) for k in range(4)}
           for i in range(len(COLUMNS)) for j in range(len(ROWS) - 3)] \
        + [{(COLUMNS[i+k], ROWS[j+k]) for k in range(4)}
           for i in range(len(COLUMNS) - 3) for j in range(len(ROWS) - 3)] \
        + [{(COLUMNS[i+k], ROWS[j-k]) for k in range(4)}
           for i in range(len(COLUMNS) - 3) for j in range(3, len(ROWS))]

def who_is_winner(pieces_positions):
    players = {}
    board = dict.fromkeys(COLUMNS, 0)
    for position in pieces_positions:
        column, player = position.split('_')
        pos = (column, board[column])
        board[column] += 1
        players.setdefault(player, set()).add(pos)
        if any(line <= players[player] for line in LINES):
            return player
    return "Draw"

# test
who_is_winner([
  "C_Yellow", "E_Red", "G_Yellow", "B_Red", "D_Yellow", "B_Red", "B_Yellow", "G_Red", "C_Yellow", "C_Red",
  "D_Yellow", "F_Red", "E_Yellow", "A_Red", "A_Yellow", "G_Red", "A_Yellow", "F_Red", "F_Yellow", "D_Red",
  "B_Yellow", "E_Red", "D_Yellow", "A_Red", "G_Yellow", "D_Red", "D_Yellow", "C_Red"
])

# if __name__ == "__main__":


# -*- coding: utf-8 -*-

from django.conf.urls.i18n import i18n_patterns
from django.template.loader import render_to_string
from django.http import HttpResponse, HttpResponseNotFound, HttpResponseServerError
from django.conf.urls import patterns, include, url
from views.index    import IndexView, RegisterView
from views.captcha  import CaptchaView
from views.action   import ActionView
from views.addedit  import AddView, EditView
from views.news     import NewsView
from views.article  import ArticleView, ArticleEditView, CategoryView, ArticleReviewEditsView
from views.search   import SearchView
from views.base import menu as m

from django.contrib import admin
admin.autodiscover()

def err404(request):
    menu = m()
    if request.method == 'GET':
        return HttpResponseNotFound(render_to_string('404.html', locals()))
    else:
        return HttpResponseNotFound('404')

def err500(request):
    menu = m()
    if request.method == 'GET':
        return HttpResponseServerError(render_to_string('500.html', locals()))
    else:
        return HttpResponseServerError('500')

handler404 = 'urls.err404'
handler500 = 'urls.err500'

urlpatterns = i18n_patterns(
    '',
    url(r'^$', IndexView.as_view()),
    url(r'^news/(?P<id>\d+)$',       NewsView.as_view()),

    url(r'^news$',      IndexView.as_view()),
    url(r'^news/$',     IndexView.as_view()),
    url(r'^news/add$',               AddView.as_view()),
    url(r'^news/(?P<id>\d+)/edit$',  EditView.as_view()),
    url(r'^news/(?P<id>\d+)$',       NewsView.as_view()),
    url(r'^news/page(?P<page>\d+)$', IndexView.as_view()),

    url(r'^login$', RegisterView.as_view()),

    url(r'^article/cat/(?P<title>.+)$',          CategoryView.as_view()),
    url(r'^article/(?P<title>.+)/edit$',         ArticleEditView.as_view()),
    #url(r'^article/(?P<title>.+)/review/edits$', ArticleReviewEditsView.as_view()),
    url(r'^article/(?P<title>.+)$',              ArticleView.as_view()),

    url(r'^search$',     SearchView.as_view()),
)

urlpatterns += patterns(
    '',
    url(r'^$',          IndexView.as_view()),
    url(r'^act$',     ActionView.as_view()),
    url(r'^news/action$',            ActionView.as_view()),

    url(r'^captcha/1\.jpg$',      CaptchaView.as_view()),

    # Uncomment the admin/doc line below to enable admin documentation:
    # url(r'^admin/doc/', include('django.contrib.admindocs.urls')),

    # Uncomment the next line to enable the admin:
     url(r'^admin/', include(admin.site.urls)),
)


from domain.entitity import player

class cons(object):
    '''
    This is the console module.
    '''
    def __init__(self, service):
        self.__service = service
        self.__commands = {
            "print"   : self.ui_print,
            "medals"  : self.ui_medals,
            "stars"   : self.ui_stars
            }

    def ui_stars(self,params):
        self.__service.writeStars()

    def ui_print(self,params):
        list = self.__service.getPlayers()
        for el in list:
            print(str(el) + "," + str(el.get_distance()))

    def ui_medals(self,params):
        list = self.__service.getPlayers()
        list.sort( key = player.get_distance, reverse = True )
        place = 1
        for el in list:
            if place == 3 :
                print("Bronze medal goes to :")
                print(el.get_name() + "," + str(el.get_distance()))
                place += 1
            if place == 2 :
                print("Silver medal goes to :")
                print(el.get_name() + "," + str(el.get_distance()))
                place += 1
            if place == 1 :
                print("Gold medal goes to :")
                print(el.get_name() + "," + str(el.get_distance()))
                place += 1

    def run(self):
        while True:
            raw_cmd = input("Enter command >> ")
            raw_cmd.strip()
            cmd = raw_cmd.split(" ")
            if cmd[0] == "exit":
                print("The program will be terminated ! Bye.")
                return
            elif cmd[0] == "print":
                self.ui_print(cmd[1:])
            elif cmd[0] == "medals":
                self.ui_medals(cmd[1:])
            elif cmd[0] == "stars" :
                self.ui_stars(cmd[1:])
            else:
                print("Invalid command !\n")


import OpenGL.GL as gl
import OpenGL.GLUT as glut
import sys
from utility.os_interface import read_file_data
import numpy
from numpy.lib.arraypad import np
import ctypes
import log

class Viewer:
    point_count = 0

    def __init__(self, points, colors, file_iso_function, scale, delta, target_value, vertex_file, time):
        self.point_count = len(points)

        self._init_glut(time)

        # read data
        data = numpy.zeros(self.point_count, dtype=[("position", np.float32, 2),
                                                    ("color", np.float32, 4)])
        data['color'] = colors
        data['position'] = points

        # read shaders
        path = "./shader"
        vertex_code = read_file_data(path, vertex_file)
        fragment_code = read_file_data(path, "fragment1.glsl")

        buffer = self._build_buffer(data)  # TODO ?
        program = self._init_program(vertex_code, fragment_code)
        self._bind_buffers(program, data, buffer)

        # bind uniforms
        self._bind_scale(program, scale)
        self._bind_delta(program, delta)
        self._bind_projection(program)
        self._bind_modelview(program)
        self._bind_target_value(program, target_value)

        glut.glutMainLoop()

    def _display(self):
        gl.glClear(gl.GL_COLOR_BUFFER_BIT)
        # gl.glDrawArrays(gl.GL_TRIANGLE_STRIP, 0, 4)

        gl.glEnable(gl.GL_PROGRAM_POINT_SIZE)
        gl.glClearColor(1.0, 1.0, 1.0, 1.0)

        # for x in range(0, self.point_count/2, 2):
        #  gl.glDrawArrays(gl.GL_LINES, x, x+1)

        gl.glDrawArrays(gl.GL_POINTS, 0, self.point_count)
        glut.glutSwapBuffers()

    def _set_window_size(self, width, height):
        gl.glViewport(0, 0, width, height)

    def _keyboard(self, key, x, y):
        if key == '\033':
            sys.exit()

    def _init_glut(self, time):

        if type(time) != bytes:
            raise TypeError

        glut.glutInit()
        glut.glutInitDisplayMode(glut.GLUT_DOUBLE | glut.GLUT_RGBA)
        glut.glutCreateWindow(time)
        glut.glutReshapeWindow(700, 700)
        glut.glutReshapeFunc(self._set_window_size)
        glut.glutDisplayFunc(self._display)
        glut.glutKeyboardFunc(self._keyboard)

    def _init_program(self, vertex_code, fragment_code):
        program = gl.glCreateProgram()
        print(gl.glGetError())
        vertex = gl.glCreateShader(gl.GL_VERTEX_SHADER)
        print(gl.glGetError())
        fragment = gl.glCreateShader(gl.GL_FRAGMENT_SHADER)
        print(gl.glGetError())
        # Set shaders source
        gl.glShaderSource(vertex, vertex_code)
        print(gl.glGetError())
        gl.glShaderSource(fragment, fragment_code)
        print(gl.glGetError())
        # Compile shaders
        gl.glCompileShader(vertex)
        print(gl.glGetError())
        gl.glCompileShader(fragment)
        print(gl.glGetError())

        # We can now build and link the program

        gl.glAttachShader(program, vertex)
        print(gl.glGetError())
        gl.glAttachShader(program, fragment)
        print(gl.glGetError())
        gl.glLinkProgram(program)
        print(gl.glGetError())

        # We can not get rid of shaders, they won't be used again:

        gl.glDetachShader(program, vertex)
        print(gl.glGetError())
        gl.glDetachShader(program, fragment)
        print(gl.glGetError())

        # Finally, we make program the default program to be ran. We can do it now because we'll use a single in this example:
        print(glut.glutReportErrors())
        if gl.glGetShaderiv(vertex, gl.GL_COMPILE_STATUS) == 0:
            print(gl.glGetShaderInfoLog(vertex))

        if gl.glGetShaderiv(fragment, gl.GL_COMPILE_STATUS) == 0:
            print(gl.glGetShaderInfoLog(fragment))

        if gl.glGetProgramiv(program, gl.GL_LINK_STATUS) == 0:
            print(gl.glGetProgramInfoLog(program))

        gl.glUseProgram(program)
        print(gl.glGetError())

        return program

    def _build_buffer(self, data):
        # Request a buffer slot from GPU
        buffer = gl.glGenBuffers(1)

        # Make this buffer the default one
        gl.glBindBuffer(gl.GL_ARRAY_BUFFER, buffer)

        # Upload data
        gl.glBufferData(gl.GL_ARRAY_BUFFER, data.nbytes, data, gl.GL_DYNAMIC_DRAW)

        return buffer

    def _bind_buffers(self, program, data, buffer):
        stride = data.strides[0]
        offset = ctypes.c_void_p(0)
        loc = gl.glGetAttribLocation(program, "position")

        gl.glEnableVertexAttribArray(loc)
        gl.glBindBuffer(gl.GL_ARRAY_BUFFER, buffer)

        gl.glVertexAttribPointer(loc, 3, gl.GL_FLOAT, False, stride, offset)

        offset = ctypes.c_void_p(data.dtype["position"].itemsize)
        loc = gl.glGetAttribLocation(program, "color")

        gl.glEnableVertexAttribArray(loc)
        gl.glBindBuffer(gl.GL_ARRAY_BUFFER, buffer)
        gl.glVertexAttribPointer(loc, 4, gl.GL_FLOAT, False, stride, offset)

    # Bind Uniform:
    def _bind_scale(self, program, scale):
        loc = gl.glGetUniformLocation(program, "scale")
        gl.glUniform1f(loc, scale)

    def _bind_target_value(self, program, target_value):
        loc = gl.glGetUniformLocation(program, "target_value")
        gl.glUniform1f(loc, target_value)

    def _bind_delta(self, program, delta):
        loc = gl.glGetUniformLocation(program, "delta")
        gl.glUniform1f(loc, delta)

    def _bind_projection(self, program):
        loc = gl.glGetUniformLocation(program, "in_projection_matrix")
        projection = np.matrix([1, 1, 1, 1])
        projection = [1, 1, 1, 1]
        # GLint location, GLsizei count, GLboolean transpose,  	const GLfloat *value
        gl.glUniformMatrix4fv(loc, len(projection), True, projection)

    def _bind_modelview(self, program):
        # loc = gl.glGetUniformLocation(program, "in_modelview_matrix")
        # projection = np.matrix([1, 1, 1, 1])
        # gl.glUniformMatrix4fv(loc, projection)
        return


from flask import render_template, flash, redirect, url_for, request, jsonify, \
    current_app, g
from flask_babelex import lazy_gettext as _
from flask_login import current_user, login_required, login_user, logout_user
from rapidannotator import db
from rapidannotator import bcrypt
from rapidannotator.modules.clustering import blueprint
import datetime, os, base64, json, pandas
from rapidannotator.models import User, Experiment, Clustering, NotificationInfo
import requests as rq

@blueprint.route('/_setJob', methods=['GET', 'POST'])
def _setJob():
	experimentId = int(request.args.get('experimentId', None))
	userId = int(request.args.get('userId', None))

	clustering = Clustering.query.filter_by(experiment_id = experimentId).first()
	if clustering is None:
		clustering = Clustering(experiment_id = experimentId, user_id = userId, status=0)
		db.session.add(clustering)
		db.session.commit()

		response = {}
		response['success'] = True
		return jsonify(response)
	else:
		clustering = Clustering.query.filter_by(experiment_id = experimentId).first()
		response = {}
		response['success'] = False
		response['msg'] = "Clustering is under process ! Please Wait!"
		if clustering.status == 2:
			response['msg'] = "Clustering is already Done!"
		return jsonify(response)

@blueprint.route('/getJobData', methods=['GET'])
def index():
	clusters = Clustering.query.all()
	response = {}
	response['jobsData'] = []
	for cluster in clusters:
		if str(cluster.status) == '0':
			from rapidannotator import app
			experimentDIR = os.path.join(app.config['UPLOAD_FOLDER'], str(cluster.experiment_id))
			inputConcordance = os.path.join(experimentDIR, 'concordance.csv')
			data = pandas.read_csv(inputConcordance)
			first_pair = data['Screenshot'].tolist()
			second_pair = list(range(1, len(first_pair) + 1))
			dictionary = {'fileId': second_pair, 'imageURLS': first_pair, 'jobId': cluster.id, 'experiment_id': str(cluster.experiment_id)}
			response['jobsData'].append(dictionary)
	response['success'] = True
	return jsonify(response)

@blueprint.route('/setJobStatus', methods=['GET', 'POST'])
def setJobStatus():

	content = request.data
	content = content.decode('utf-8')
	content = eval(content)
	jobId = int(content['jobId'])

	clustering = Clustering.query.filter_by(id = jobId).first()
	if clustering is None:
		response = {}
		response['success'] = False
		return jsonify(response)

	if content['jobStatus'] == 'Processing':
		clustering.status = 1
		db.session.commit()

	response = {}
	response['success'] = True
	return jsonify(response)

@blueprint.route('/publishResults', methods=['GET', 'POST'])
def publishResults():

	content = request.data
	content = content.decode('utf-8')
	content = eval(content)

	from rapidannotator import app
	experimentDIR = os.path.join(app.config['UPLOAD_FOLDER'], str(content['experiment_id']))
	outJson = os.path.join(experimentDIR, 'output.json')

	out = content['largest1']
	out1 = sorted(range(len(out)), key=lambda k: out[k], reverse=True)
	content['sortOrder'] = out1

	with open(outJson, 'w') as json_file:
		json.dump(content, json_file, indent = 4, sort_keys=True)

	jobId = int(content['job_id'])
	clustering = Clustering.query.filter_by(id = jobId).first()

	if clustering is None:
		response = {}
		response['success'] = False
		return jsonify(response)

	clustering.status = 2
	db.session.commit()

	experimentId = int(content['experiment_id'])
	experiment_info = Experiment.query.filter_by(id=experimentId).first()
	experiment_owners = experiment_info.owners
	user = User.query.filter_by(id = int(clustering.user_id)).first()

	for owner in experiment_owners:
		owner.numNotif += 1
		message = 'The Clustering for the experiment ' + experiment_info.name + ' got completed by the ' + user.fullname
		notify = NotificationInfo()
		notify.user_id = owner.id
		notify.username = user.username
		notify.notification = message
		db.session.add(notify)
		db.session.commit()

	response = {}
	response['success'] = True
	return jsonify(response)

@blueprint.route('/getStatus', methods=['GET', 'POST'])
def getStatus():
	experiment_id = request.form['experiment_id']
	if experiment_id is not None:
		experiment_id = int(experiment_id)

	clustering = Clustering.query.filter_by(experiment_id = experiment_id, user_id = int(current_user.id)).first()
	if clustering is None:
		status = 0
	else:
		status = clustering.status

	response = {}
	response['success'] = True
	response['status'] = status
	return jsonify(response)

@blueprint.route('/toggleDisplay', methods=['GET', 'POST'])
def toggleDisplay():
	experiment_id = request.args.get('experiment_id', None)
	option = request.args.get('option')
	clustering = Clustering.query.filter_by(experiment_id = experiment_id, user_id = int(current_user.id)).first()
	response = {}
	if clustering is None:
		response['success'] = False
	else:
		if option == "Yes":
			clustering.display = 1
		else:
			clustering.display = 0
		db.session.commit()
		response['success'] = True
	return jsonify(response)


#Program to find the number of occurrence of letters in a string
def most_frequent(str1):
    l=len(str1)
    for i in range(0,l):
        cnt=0
        for j in range(0,l):
            if str1[i]==str1[j]:
                cnt+=1
        occ[str1[i]]=str(cnt)
    print(sorted(occ.items(), key=lambda x: x[1], reverse=True))
occ={}
str_name=input("Please enter a String: ")
most_frequent(str_name)


# myproject.myapi.utils.py
from channels.auth import AuthMiddlewareStack
from channels.db import database_sync_to_async
from django.contrib.auth.models import AnonymousUser

from rest_framework.authtoken.models import Token
from django.db import close_old_connections

@database_sync_to_async
def get_user(token_key):
    try:
        return Token.objects.get(key=token_key).user
    except Token.DoesNotExist:
        return AnonymousUser()

class TokenAuthMiddleware:
    """
    Token authorization middleware for Django Channels 2
    see:
    https://channels.readthedocs.io/en/latest/topics/authentication.html#custom-authentication
    """

    def __init__(self, inner):
        self.inner = inner

    def __call__(self, scope):
        return TokenAuthMiddlewareInstance(scope, self)

class TokenAuthMiddlewareInstance:
    def __init__(self, scope, middleware):
        self.middleware = middleware
        self.scope = dict(scope)
        self.inner = self.middleware.inner

    async def __call__(self, receive, send):
        close_old_connections()
        headers = dict(self.scope["headers"])
        print(headers[b"cookie"])
        if b"authorization" in headers[b"cookie"]:
            print('still good here')
            cookies = headers[b"cookie"].decode()
            token_key = re.search("authorization=(.*)(; )?", cookies).group(1)
            if token_key:
                self.scope["user"] = await get_user(token_key)

        # inner = self.inner(self.scope)
        return await self.inner(self.scope,receive, send)

TokenAuthMiddlewareStack = lambda inner: TokenAuthMiddleware(AuthMiddlewareStack(inner))


from signer import *
from optparse import OptionParser
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.utils import Sequence
from keras.optimizers import Adam
from keras.callbacks import Callback,ModelCheckpoint
from sklearn.externals import joblib

pd.set_option('display.max_rows',None)

op = OptionParser(usage='python3 poc_eval_mp.py [options] [objs-folder] [anit-folder] [meta_embed_file] [model_dest]')
op.add_option('--target_test', dest='target_test', help='an additional target test set for evaluation, used while test_ratio is 0')
op.add_option('--anti_test', dest='anti_test', help='an addtional anit test set for evaluation, used while test_ratio is 0')
op.add_option('--bec_test', dest='bec_test', help='an addtional bec test set for evaluation, used while test_ratio is 0')
op.add_option('--test_ratio', dest='test_ratio', type='float', default=0, help='a ratio for ratio of test set, default is 0')
op.add_option("-v", dest="verbose", action='store_true', default=False, help='a switch to turn on/off detail messages')
op.add_option('-n', dest='enable_nmlzr', action='store_true', default=False, help='a switch to turn on/off the normalizer')
op.add_option('--expt_debug', dest='expt_debug', action='store_true', default=False, help='experiment mode')
options,args  = op.parse_args()

EXPT_DEBUG  = options.expt_debug

#-- Get Data Paths --#
tr_target_path  = args[0]
tr_anti_path    = args[1]
model_path      = args[2] if len(args)>2 else None
tt_target_path  = options.target_test
tt_anti_path    = options.anti_test
tt_bec_path     = options.bec_test

def meta_to_word(t,ignore_length=False,only_initial=False,only_final=False,only_length=False):
    t  = [('<!--signature-->',0) if x=='<!--signature-->' else (list(x.keys())[0],list(x.values())[0]) for x in t]
    if only_initial:
        t  = [x[0] for x,y in t]
    elif only_final:
        t  = [x[-1] for x,y in t]
    elif only_length:
        t  = [x if len(x)==1 else (str(y) if y<31 else '31+') for x,y in t]
    elif ignore_length:
        t  = [x for x,y in t]
    else:
        t  = [x if y<=len(x) else x[0]+'_'*(y-len(x))+x[1:] for x,y in t]
    t  = '<sep>'.join(t)
    return t

def read_meta_path(meta_path):
    filelist  = sorted(glob.glob(os.path.join(meta_path,'*.meta')))
    text      = pd.DataFrame(index=pd.Index([],name='title'))
    for f in filelist:
        title  = os.path.splitext(os.path.split(f)[1])[0]
        t      = read_meta(f)
        text.loc[title,'filename'] = f
        text.loc[title,'raw']  = json.dumps(t,separators=(',',':'))
        text.loc[title,'wrd']  = meta_to_word(t)
        text.loc[title,'wrd_nolen']  = meta_to_word(t,ignore_length=True)
        text.loc[title,'ch0']  = meta_to_word(t,only_initial=True)
        text.loc[title,'ch1']  = meta_to_word(t,only_final=True)
        text.loc[title,'len']  = meta_to_word(t,only_length=True)
    return text

#-- Read Meta --#
tr_target  = read_meta_path(tr_target_path) if tr_target_path else None
tr_anti    = read_meta_path(tr_anti_path) if tr_anti_path else None
tt_target  = read_meta_path(tt_target_path) if tt_target_path else None
tt_anti    = read_meta_path(tt_anti_path) if tt_anti_path else None
tt_bec     = read_meta_path(tt_bec_path) if tt_bec_path else None

#-- Compile Training and Test Data --#
data_tr    = pd.concat([tr_target,tr_anti],0,keys=['target','anti'],names=['grp',],sort=False)
data_tr['bec']  = data_tr.index.get_level_values('grp')!='target'
data_tt    = pd.concat([tt_target,tt_anti,tt_bec],0,keys=['target','anti','bec'],names=['grp',],sort=False)
data_tt['bec']  = data_tt.index.get_level_values('grp')!='target'

#-- Fit Tokenizers --#
MAX_NUM_WORDS  = 20000
tokenizer  = {}
for col in ('wrd','wrd_nolen','ch0','ch1','len'):
    tokenizer[col]  = Tokenizer(num_words=MAX_NUM_WORDS,filters='',lower=False,split='<sep>',char_level=False,oov_token=None)
    tokenizer[col].fit_on_texts(data_tr[col].str.replace('\r\n','\n'))

#-- Text to Tokenizer Word Index --#
X_tr  = {}
for col in ('wrd','wrd_nolen','ch0','ch1','len'):
    X_tr[col]  = tokenizer[col].texts_to_sequences(data_tr[col].str.replace('\r\n','\n'))

max_seq_len = min(max([len(x) for x in X_tr['wrd']]),1000)
for col in ('wrd','wrd_nolen','ch0','ch1','len'):
    X_tr[col]  = pad_sequences(X_tr[col],maxlen=max_seq_len,dtype='int32',padding='post',truncating='post',value=0)

y_tr  = data_tr.bec

X_tt  = {}
for col in ('wrd','wrd_nolen','ch0','ch1','len'):
    X_tt[col]  = tokenizer[col].texts_to_sequences(data_tt[col].str.replace('\r\n','\n'))
    X_tt[col]  = pad_sequences(X_tt[col],maxlen=max_seq_len,dtype='int32',padding='post',truncating='post',value=0)

y_tt  = data_tt.bec

def token_embedding_mc(max_seq_len,tokenizer,channels,output_dim,mask_zero=False):
    embed_model  = {}
    for col in channels:
        embed_model[col]  = token_embedding(max_seq_len,tokenizer[col],output_dim=output_dim,mask_zero=mask_zero)
    #
    embed_mc_output   = Concatenate(axis=2)([embed_model[col].output for col in channels])
    return Model(inputs=[embed_model[col].input for col in channels],outputs=embed_mc_output)

#-----------#
#-- CNN 7 --#
#-----------#
#---- Preprocess: All characters
#---- Channels: (wrd,wrd_nolen,ch0,ch1,len)
#---- Seed: 0
#---- ND: 20, FS: (3,4,5), NF: 64, Dropout: 0.2, Lambda: 0.001, Loss: 'mean_squared_error', AdamDecay: 0.02, MaxEpochs: fr:55 jp:60, BatchSize: 256
keras_seed(0)
MAX_EPOCHS   = 60 if tt_target_path.split(os.sep)[4]=='Trend_jp' else 55
CHANNELS     = ('wrd','wrd_nolen','ch0','ch1','len')
CNN_DROPOUT  = 0.2
REG_LAMBDA   = 0.001
ADAM_DECAY   = 0.02
embed_model_mc    = token_embedding_mc(max_seq_len,tokenizer,channels=CHANNELS,output_dim=20)
tweet_embed_model = YoonKimCNN(embed_model_mc,filter_sizes=(3,4,5),num_filters=64,batch_norm=False,dropout=CNN_DROPOUT)
preds  = Dense(1,activation='sigmoid',
    kernel_regularizer=regularizers.l2(REG_LAMBDA),
    bias_regularizer=regularizers.l2(REG_LAMBDA),
    )(tweet_embed_model.output)
model  = Model(inputs=tweet_embed_model.inputs,outputs=preds)
model.compile(loss='mean_squared_error', #'binary_crossentropy', #
    optimizer=Adam(decay=ADAM_DECAY),
    metrics=['accuracy'])

if EXPT_DEBUG: t0 = time.time()

out  = model.fit([X_tr[col] for col in CHANNELS],y_tr,validation_data=([X_tt[col] for col in CHANNELS],y_tt),epochs=MAX_EPOCHS,batch_size=256,class_weight=None,shuffle=True,verbose=int(EXPT_DEBUG))
hist = pd.DataFrame(out.history,index=range(1,MAX_EPOCHS+1))[['loss','acc','val_loss','val_acc']]
if EXPT_DEBUG:
    print(time.time() - t0)
    print(hist[(hist.index.isin(range(0,MAX_EPOCHS+1,5)))|(hist.val_loss==hist.val_loss.min())|(hist.val_acc==hist.val_acc.max())])

data_tr['bec_prob']  = model.predict([X_tr[col] for col in CHANNELS])
data_tt['bec_prob']  = model.predict([X_tt[col] for col in CHANNELS])

# #--------------------------------------------------------#
# #-- CNN 8 - Baseline CNN w/ space char and len channel --#
# #--------------------------------------------------------#
# #---- Preprocess: All characters
# #---- Channels: (wrd_nolen,len)
# #---- Seed: 0
# #---- ND: 20, FS: (3,4,5), NF: 32, Dropout: 0.02, Lambda: 0.001, Loss: 'binary_crossentropy', AdamDecay: 0, MaxEpochs: fr:85 jp:70, BatchSize: 256
# keras_seed(0)
# MAX_EPOCHS   = 70 if tt_target_path.split(os.sep)[4]=='Trend_jp' else 85
# CHANNELS     = ('wrd_nolen','len')
# CNN_DROPOUT  = 0.02
# REG_LAMBDA   = 0.001
# embed_model_mc    = token_embedding_mc(max_seq_len,tokenizer,channels=CHANNELS,output_dim=20)
# tweet_embed_model = YoonKimCNN(embed_model_mc,filter_sizes=(3,4,5),num_filters=32,batch_norm=False,dropout=CNN_DROPOUT)
# preds  = Dense(1,activation='sigmoid',
#     kernel_regularizer=regularizers.l2(REG_LAMBDA),
#     bias_regularizer=regularizers.l2(REG_LAMBDA),
#     )(tweet_embed_model.output)
# model  = Model(inputs=tweet_embed_model.inputs,outputs=preds)
# model.compile(loss='binary_crossentropy', #'mean_squared_error', #
#     optimizer='adam',
#     metrics=['accuracy'])

# if EXPT_DEBUG: t0 = time.time()

# out  = model.fit([X_tr[col] for col in CHANNELS],y_tr,validation_data=([X_tt[col] for col in CHANNELS],y_tt),epochs=MAX_EPOCHS,batch_size=256,class_weight=None,shuffle=True,verbose=int(EXPT_DEBUG))
# hist = pd.DataFrame(out.history,index=range(1,MAX_EPOCHS+1))[['loss','acc','val_loss','val_acc']]
# if EXPT_DEBUG:
#     print(time.time() - t0)
#     print(hist[(hist.index.isin(range(0,MAX_EPOCHS+1,5)))|(hist.val_loss==hist.val_loss.min())|(hist.val_acc==hist.val_acc.max())])

# data_tr['bec_prob']  = model.predict([X_tr[col] for col in CHANNELS])
# data_tt['bec_prob']  = model.predict([X_tt[col] for col in CHANNELS])

#----------------#
#-- Save Model --#
#----------------#
if model_path:
    joblib.dump({'tokenizer':tokenizer,'hist':hist,},os.path.join(model_path,tr_anti_path+'_cnn7.pkl'))
    model.save(os.path.join(model_path,tr_anti_path+'_cnn7.hd5'),overwrite=True)

#---------------------------#
#-- Evaluation and Output --#
#---------------------------#
BEC_THD  = 0.3
data_tr['bec_hat']   = data_tr.bec_prob>BEC_THD
data_tt['bec_hat']   = data_tt.bec_prob>BEC_THD
res  = pd.concat([
    scores(data_tr.bec,data_tr.bec_hat),
    scores(data_tt.bec,data_tt.bec_hat),
    ],1,keys=('tr','tt'))
if EXPT_DEBUG: print(res)

# res  = pd.Series([
#     accuracy_score(data_tr.loc['target'].bec,data_tr.loc['target'].bec_hat),
#     accuracy_score(data_tr.loc['anti'].bec,data_tr.loc['anti'].bec_hat),
#     accuracy_score(data_tt.loc['target'].bec,data_tt.loc['target'].bec_hat),
#     accuracy_score(data_tt.loc['anti'].bec,data_tt.loc['anti'].bec_hat),
#     ],index=('tr_target','tr_anti','tt_target','tt_anti'))

#-- Output Prediction Results to Stdout --#
if not EXPT_DEBUG:
    # data_tt['filename']  = 'k-fold' + data_tt.filename.str.split('k-fold').str[1]
    for idx,row in data_tt.iterrows():
        print("%(filename)s,%(bec_prob)f,%(bec_hat)d"%row)


# default imports
from sklearn.model_selection import train_test_split
from sklearn.tree import DecisionTreeClassifier
from sklearn.metrics import accuracy_score
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np

data = pd.read_csv("./data/house_pricing.csv")
X = data.iloc[:, :-1]
y = data.iloc[:, -1]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=9)

depth_list =  [8, 10, 15, 20, 50, 100, 120, 150, 175, 200]
def decision_classifier_plot(X_train, X_test, y_train, y_test,depth_list):
    test_score=[]
    train_score=[]
# Write your solution here :
    for i in range(len(depth_list)):
        model =DecisionTreeClassifier(max_depth=depth_list[i])
        model.fit(X_train,y_train)
        y_pred=model.predict(X_train)
        y_pred_test=model.predict(X_test)
        train_score.insert(i,accuracy_score(y_train,y_pred))
        test_score.insert(i,accuracy_score(y_test,y_pred_test))
    plt.plot(depth_list, train_score, 'b', label='train set')
    plt.plot(depth_list, test_score, 'g', label='test set')
    plt.legend(loc='best')
    plt.xlabel('depth')
    plt.ylabel('accuracy_score')
    plt.show()


#!/usr/bin/env python
# coding: utf-8

# In[1]:

#importing libraries:
import numpy as np
from flask import Flask, request, jsonify, render_template
import pickle

# In[2]:

app=Flask(__name__)

# In[3]:

model=pickle.load(open('log_model.pkl','rb'))

# In[4]:

@app.route('/')
def home():
    return render_template('individual.html')

#%%
term_to_int={0:"36 Months",1:"64 Months"}

application_type={0:"Individual"}

grade={0:"A",1:"B",2:"C",3:"D",4:"E",5:"F",6:"G"}

sub_grade={0:"A1",1:"A2",2:"A3",3:"A4",4:"A5",
           5:"B1",6:"B2",7:"B3",8:"B4",9:"B5",
           10:"C1",11:"C2",12:"C3",13:"C4",14:"C5",
           15:"D1",16:"D2",17:"D3",18:"D4",19:"D5",
           20:"E1",21:"E2",22:"E3",23:"E4",24:"E5",
           25:"F1",26:"F2",27:"F3",28:"F4",29:"F5",
           30:"G1",31:"G2",32:"G3",33:"G4",34:"G5"}

emp_length={0:"1 Year",1:"10+ Years",2:"2 Years",3:"3 Years",4:"4 Years",5:"5 Year",
            6:"6 Years",7:"7 Years",8:"8 Years",9:"9 Years",10:"< 1 Year"}

home_ownership={0:"Mortage",1:"Others",2:"Own",3:"Rent"}

verification_status={0:"Income Source Verified",1:"Not Verified",2:"Verified"}

purpose={0:"Car",1:"Credit Card",2:"Dept Consolidation",3:"Educational",4:"Home Improvement",
         5:"House",6:"Major Purpose",7:"Medical",8:"Moving",9:"Others",10:"Renewable Energy",
         11:"Small Business",12:"Vacation",13:"Wedding"}

addr_state={0:"Alaska",1:"Alabama",2:"Arkansas",3:"Arizona",4:"California",5:"Colorado",
            6:"Connecticut",7:"District of Columbia",8:"Delaware",9:"Florida",10:"Georgia",
            11:"Hawaii",12:"Iowa",13:"Idaho",14:"Illinois",15:"Indiana",16:"Kansas",
            17:"Kentucky",18:"Louisiana",19:"Massachusetts",20:"Maryland",21:"Maine",
            22:"Michigan",23:"Minnesota",24:"Missouri",25:"Mississippi",26:"Montana",
            27:"North Carolina",28:"North Dakota",29:"Nebraska",30:"New Hampshire",
            31:"New Jersey",32:"New Mexico",33:"Nevada",34:"New York",35:"Ohio",36:"Oklahoma",
            37:"Oregon",38:"Pennsylvania",39:"Rhode Island",40:"South Carolina",41:"South Dakota",
            42:"Tennessee",43:"Texas",44:"Utah",45:"Virginia",46:"Vermont",47:"Washington",
            48:"Wisconsin",49:"West Virginia",50:"Wyoming"}

initial_list_status={0:"Fractional",1:"Whole"}
# In[5]:

@app.route("/Individual", methods=['POST'])
def Individual():
    int_features=[int(x) for x in request.form.values()]
    final_features=[np.array(int_features)]
    prediction=model.predict(final_features)

    data2=request.form["loan_amnt"]
    data3=request.form["term"]
    data4=request.form["int_rate"]
    data5=request.form["installment"]
    data6=request.form["grade"]
    data7=request.form["sub_grade"]
    data8=request.form["emp_length"]
    data9=request.form["home_ownership"]
    data10=request.form["annual_inc"]
    data11=request.form["verification_status"]
    data12=request.form["purpose"]
    data13=request.form["addr_state"]
    data14=request.form["dti"]
    data15=request.form["delinq_2yrs"]
    data16=request.form["inq_last_6mths"]
    data17=request.form["open_acc"]
    data18=request.form["pub_rec"]
    data19=request.form["revol_bal"]
    data20=request.form["revol_util"]
    data21=request.form["total_acc"]
    data22=request.form["initial_list_status"]
    data23=request.form["out_prncp"]
    data24=request.form["out_prncp_inv"]
    data25=request.form["total_pymnt"]
    data26=request.form["total_rec_prncp"]
    data27=request.form["total_rec_int"]
    data28=request.form["total_rec_late_fee"]
    data29=request.form["last_pymnt_amnt"]
    data30=request.form["collections_12_mths_ex_med"]
    data31=request.form["application_type"]
    data32=request.form["acc_now_delinq"]
    data33=request.form["tot_coll_amt"]
    data34=request.form["tot_cur_bal"]

    #create original output dict
    output_dict= dict()
    output_dict['Desired Loan Amount']=data2
    output_dict['Term (in Months)']=term_to_int[int(data3)]
    output_dict['Interest Rate on the loan']=data4
    output_dict['Installment']=data5
    output_dict['Grade']=grade[int(data6)]
    output_dict['Sub Grade']=sub_grade[int(data7)]
    output_dict['Employee Length']=emp_length[int(data8)]
    output_dict['Home Ownership']=home_ownership[int(data9)]
    output_dict['Annual Income']=data10
    output_dict['Verification Status']=verification_status[int(data11)]
    output_dict['Loan Purpose']=purpose[int(data12)]
    output_dict['Address State']=addr_state[int(data13)]
    output_dict['Total Debt Obligation Income']=data14
    output_dict['Delinquency for the past 2 years']=data15
    output_dict['Inquiries in past 6 months (excluding auto and mortgage inquiries)']=data16
    output_dict["Number of open credit lines in the borrower's credit file"]=data17
    output_dict['Number of derogatory public records']=data18
    output_dict['Total credit revolving balance']=data19
    output_dict['Revolving line utilization rate']=data20
    output_dict["Total number of credit lines currently in the borrower's credit file"]=data21
    output_dict['Initial List Status']=initial_list_status[int(data22)]
    output_dict['Remaining outstanding principal for total amount funded']=data23
    output_dict['Remaining outstanding principal for portion of total amount funded by investors']=data24
    output_dict['Payments received to date for total amount funded']=data25
    output_dict['Principal received to date']=data26
    output_dict['Interest received to date']=data27
    output_dict['Late fees received to date']=data28
    output_dict['Last total payment amount received']=data29
    output_dict['Number of collections in 12 months excluding medical collections']=data30
    output_dict['Application Type']=application_type[int(data31)]
    output_dict['Number of accounts on which the borrower is now delinquent']=data32
    output_dict['Total collection amounts ever owed']=data33
    output_dict['Total current balance of all accounts']=data34

    return render_template('individualresult.html',original_input=output_dict,data=prediction)
# In[7]:

if __name__=='__main__':
    app.run(debug=True)

# In[ ]:


import torch
import torch.nn as nn
import torch.nn.functional as F
import dgl.nn.pytorch as dglnn
import dgl.function as fn

class WeightedSAGEConv(nn.Module):
    def __init__(self, in_feat, out_feat):
        super(WeightedSAGEConv, self).__init__()
        self.linear = nn.Linear(in_feat * 2, out_feat)

    def forward(self, g, h, weights):
        with g.local_scope():
            g.ndata['h'] = h
            g.edata['w'] = weights.float()
            g.update_all(message_func=fn.u_mul_e('h', 'w', 'm'), reduce_func=fn.mean('m', 'h_N'))
            h_N = g.ndata['h_N']
            h_total = torch.cat([h, h_N], dim=1)
            return self.linear(h_total)

class RGCNLayer(nn.Module):
    """
    R-GCN layer using basis decomposition
    """
    def __init__(self, in_feat, out_feat, num_rels, num_bases=-1, bias=None,
                 activation=None, is_input_layer=False):
        super(RGCNLayer, self).__init__()
        self.in_feat = in_feat
        self.out_feat = out_feat
        self.num_rels = num_rels
        self.num_bases = num_bases
        self.bias = bias
        self.activation = activation
        self.is_input_layer = is_input_layer

        # sanity check
        if self.num_bases <= 0 or self.num_bases > self.num_rels:
            self.num_bases = self.num_rels

        # weight bases in equation (3)
        self.weight = nn.Parameter(torch.Tensor(self.num_bases, self.in_feat,
                                                self.out_feat))
        if self.num_bases < self.num_rels:
            # linear combination coefficients in equation (3)
            self.w_comp = nn.Parameter(torch.Tensor(self.num_rels, self.num_bases))

        # add bias
        if self.bias:
            self.bias = nn.Parameter(torch.Tensor(out_feat))

        # init trainable parameters
        nn.init.xavier_uniform_(self.weight,
                                gain=nn.init.calculate_gain('relu'))
        if self.num_bases < self.num_rels:
            nn.init.xavier_uniform_(self.w_comp,
                                    gain=nn.init.calculate_gain('relu'))
        if self.bias:
            nn.init.xavier_uniform_(self.bias,
                                    gain=nn.init.calculate_gain('relu'))

    def forward(self, g):
        if self.num_bases < self.num_rels:
            # generate all weights from bases (equation (3))
            weight = self.weight.view(self.in_feat, self.num_bases, self.out_feat)
            weight = torch.matmul(self.w_comp, weight).view(self.num_rels,
                                                        self.in_feat, self.out_feat)
        else:
            weight = self.weight

        if self.is_input_layer:
            def message_func(edges):
                # for input layer, matrix multiply can be converted to be
                # an embedding lookup using source node id
                embed = weight.view(-1, self.out_feat)
                index = edges.data['rel_type'] * self.in_feat + edges.src['id']
                return {'msg': embed[index] * edges.data['norm']}
        else:
            def message_func(edges):
                w = weight[edges.data['rel_type']]
                msg = torch.bmm(edges.src['h'].unsqueeze(1), w).squeeze()
                msg = msg * edges.data['norm']
                return {'msg': msg}

        def apply_func(nodes):
            h = nodes.data['h']
            if self.bias:
                h = h + self.bias
            if self.activation:
                h = self.activation(h)
            return {'h': h}

        g.update_all(message_func, fn.sum(msg='msg', out='h'), apply_func)

class HeteroRGCNLayer(nn.Module):
    """
    original R-GCN layer derived from Kipf's GCN, can be replaced with dgl.nn.pytorch.HeteroGraphConv, but slower by 40%
    """
    def __init__(self, in_size, out_size, etypes):
        super(HeteroRGCNLayer, self).__init__()
        self.weight = nn.ModuleDict({
            name: nn.Linear(in_size, out_size) for name in etypes
        })

    def forward(self, graph, feat_dict):
        funcs = {}

        for srctype, etype, dsttype in graph.canonical_etypes:
            Wh = self.weight[etype](feat_dict[srctype])
            graph.nodes[srctype].data['Wh_%s' % etype] = Wh
            funcs[etype] = (fn.copy_u('Wh_%s' % etype, 'm'), fn.mean('m', 'h'))

        graph.multi_update_all(funcs, 'sum')

        return {ntype: graph.nodes[ntype].data['h'] for ntype in graph.ntypes}


from pygame \
    import sprite, image, transform

from core.text \
    import Text

class Inventory(sprite.Sprite):

    def __init__(self):
        super(Inventory, self).__init__()
        self.empty_path = 'sprites/ui/inventory/empty.png'
        self.full_path = 'sprites/ui/inventory/full.png'

        self.image = image.load(self.empty_path)
        self.rect = self.image.get_rect()
        self.rect.x = 10
        self.rect.y = 490
        self.item = None
        self.empty = True

    def clear(self):
        self.empty = True
        self.image.fill((255, 255, 255))
        self.image = image.load(self.empty_path)
        return self.item

    def add(self, item):
        self.empty = False
        self.item = item

        if item.name == "item":
            name = "Box"
        else:
            name = item.name

        text = Text(20, name)
        self.image = image.load(self.full_path)

        inv_w, inv_h = self.rect.size
        text_w, text_h = text.size

        img_xpos = inv_w/2 - 35/2
        img_ypos = inv_h/2 - 35/2 + 10

        text_xpos = inv_w/2 - text_w/2
        text_ypos = 2

        self.image.blit(transform.scale(item.image, (35, 35)), (img_xpos, img_ypos))
        self.image.blit(text.surface, (text_xpos, text_ypos))


import sys, string, requests, enchant,collections # requests is for HTTP requests; enchant is for checking spelling
word_repeat_limit=3
def most_popular(amount):
    # GitHub's API only allows us to return up to 100 results per page, so amount should be <= 100
    if amount > 100:
        raise Exception("amount must be less than or equal to 100")
    req = requests.get("https://api.github.com/search/repositories?q=stars:>0&per_page=" + str(amount))
    popular = [tuple(repo["full_name"].split('/')) for repo in req.json()["items"]]
    # the format of the returned value is (owner,repo)
    return popular

def get_file_type(path):
    if "." in path:
        return path.rsplit(".")[1]
    else:
        return ""

def file_paths(owner,repo,branch):
    tree = requests.get("https://api.github.com/repos/" + owner + "/" + repo + "/git/trees/"+branch+"?recursive=1").json()
    # we look for the paths of files, not directories
    files = [item["path"] for item in tree["tree"] if item["type"] == "blob"]
    # ignore hidden files
    files = [x for x in files if x[0] !="."]
    files = [x for x in files if get_file_type(x) in ["js","py","rb","php","java","rs"]] # we only can detect comments in certain file formats
    return files

def get_words(line):
    # returns a list of words in a given line (words can only include letters)
    words = filter(lambda s: s.isalpha(),line.split())
    return set(words)
counted_comment_words=collections.Counter()#keeps track of words counted
def get_word_types(text,file_type): #returns the line number and text of each single-line comment in a file
    text="\n".join(text)
    code_words=set([])
    comment_words=set([])
    line_number=1
    current_word=""
    in_code=True#This indicates what character is bounding.
    comment_type=""
    skip_next=False
    if file_type=="js" or file_type=="java" or file_type=="rs":
        for i in range(len(text)):
            if skip_next:
                skip_next=False
                continue
            char=text[i]
            if char=="\\":
                skip_next=True
            elif in_code:
                if char in string.lowercase:
                    current_word+=char
                else:
                    code_words.add(current_word)

                    current_word=""
                    if char in string.uppercase:
                        current_word+=char.lower()
                    elif char=="/" and text[i+1]=="/":
                            skip_next=True
                            in_code=False
                            comment_type="//"
                    elif char=="/" and text[i+1]=="*":
                            skip_next=True
                            in_code=False
                            comment_type="/*"
                    elif char=="\n":
                        line_number+=1
            else:
                if char.lower() in string.lowercase:
                    current_word+=char.lower()
                else:
                    if current_word!="":
                        comment_words.add((current_word,line_number))
                        counted_comment_words[current_word]+=1
                        current_word=""
                    if char=="\n" and comment_type=='//':
                        in_code=True
                        line_number+=1
                    elif char=="*" and text[i+1]=="/" and comment_type=='/*':
                        in_code=True
                        skip_next=True
                    elif char=="\n":
                        line_number+=1
    elif file_type=="php":
        for i in range(len(text)):
            if skip_next:
                skip_next=False
                continue
            char=text[i]
            if char=="\\":
                skip_next=True
            elif in_code:
                if char in string.lowercase:
                    current_word+=char
                else:
                    code_words.add(current_word)
                    current_word=""
                    if char in string.uppercase:
                        current_word+=char.lower()
                    elif char=="/" and text[i+1]=="/":
                            skip_next=True
                            in_code=False
                            comment_type="//"
                    elif char=="/" and text[i+1]=="*":
                            skip_next=True
                            in_code=False
                            comment_type="/*"
                    elif char=="#":
                            in_code=False
                            comment_type="#"
                    elif char=="\n":
                        line_number+=1
            else:
                if char.lower() in string.lowercase:
                    current_word+=char.lower()
                else:
                    if current_word!="":
                        comment_words.add((current_word,line_number))
                        counted_comment_words[current_word]+=1
                        current_word=""
                    if char=="\n" and comment_type=='//':
                        in_code=True
                        line_number+=1
                    elif char=="*" and text[i+1]=="/" and comment_type=='/*':
                        in_code=True
                        skip_next=True
                    elif char=="\n" and comment_type=='#':
                        in_code=True
                        line_number+=1
                    elif char=="\n":
                        line_number+=1
    elif file_type=="py":
        for i in range(len(text)):
            if skip_next:
                skip_next=False
                continue
            char=text[i]
            if char=="\\":
                skip_next=True
            elif in_code:
                if char in string.lowercase:
                    current_word+=char
                else:
                    code_words.add(current_word)
                    current_word=""
                    if char in string.uppercase:
                        current_word+=char.lower()
                    elif char=="#":
                            in_code=False
                            comment_type="#"
                    elif char=="\n":
                        line_number+=1
            else:
                if char.lower() in string.lowercase:
                    current_word+=char.lower()
                else:
                    if current_word!="":
                        comment_words.add((current_word,line_number))
                        counted_comment_words[current_word]+=1
                        current_word=""
                    if char=="\n" and comment_type=='#':
                        in_code=True
                        line_number+=1
                    elif char=="\n":
                        line_number+=1
    for i in counted_comment_words:
        if counted_comment_words[i]>=word_repeat_limit:#yes its 5 for now
            code_words.add(i)
    return code_words.difference([""]),comment_words

def words_in_file(text):
    wordset = set([])
    for line in text:
        wordset = wordset.union(get_words(line))
    return wordset

def get_text(owner,repo,branch,file_path):
    raw = requests.get("https://raw.githubusercontent.com/" + owner + "/" + repo + "/"+branch+"/" + file_path).text
    text = raw.split("\n")
    return text

def edits1(word): # this function is stolen from Peter Norvig's article http://norvig.com/spell-correct.html
   splits     = [(word[:i], word[i:]) for i in range(len(word) + 1)]
   deletes    = [a + b[1:] for a, b in splits if b]
   transposes = [a + b[1] + b[0] + b[2:] for a, b in splits if len(b)>1]
   replaces   = [a + c + b[1:] for a, b in splits for c in string.lowercase if b]
   inserts    = [a + c + b     for a, b in splits for c in string.lowercase]
   return set(deletes + transposes + replaces + inserts)

#from nltk.tag import pos_tag
def check_spelling(owner,repo,branch="master"):
    dictionaryUS = enchant.Dict("en_US")
    dictionaryGB = enchant.Dict("en_GB")
    paths = file_paths(owner,repo,branch)
    print("Writing down domain-specific words....")
    #special_words = [(path,words_in_file(get_text(owner,repo,branch,path))) for path in paths]
    print("Done!")
    for path in paths:
        text = get_text(owner,repo,branch,path)
        code_words,comment_words = get_word_types(text,get_file_type(path))
        excused_words = code_words
        for item in comment_words:
            word,line_number=item
            wordl = word.lower()
            if not dictionaryUS.check(wordl) and not dictionaryGB.check(wordl):
                if not word in excused_words:
                    # to narrow down the list of candidates, we only look at words that are one edit away from real words
                    edits = edits1(wordl)
                    try:
                        if True in map(lambda w: dictionaryUS.check(w), edits) or True in map(lambda w: dictionaryGB.check(w), edits):
                            url = "https://github.com/" + owner + "/" + repo + "/blob/"+branch+"/" + path + "#L" + str(line_number)
                            print(word + " from " + url)
                    except:
                        print("Error on word " + word)

def main():
    if len(sys.argv) > 3:
        check_spelling(sys.argv[1],sys.argv[2],sys.argv[3])
    elif len(sys.argv)==3:
        check_spelling(sys.argv[1],sys.argv[2])
    else:
        for (owner,repo) in most_popular(int(sys.argv[1])):
            print("Checking " + owner + "/" + repo + "....")
            check_spelling(owner,repo)

if __name__ == '__main__':
    main()


def quickSort(alist):
    quickSortHelper(alist, 0, len(alist) - 1)

def quickSortHelper(alist, first, last):
    if first < last:
        splitPoint = partition(alist, first, last)

        quickSortHelper(alist, first, splitPoint - 1)
        quickSortHelper(alist, splitPoint + 1, last)

def partition(alist, first, last):
    pivotvlue = alist[first]

    leftmark = first + 1
    rightmark = last
    done = False

    while not done:
        while alist[leftmark] <= pivotvlue and leftmark <= rightmark:
            leftmark += 1
        while alist[rightmark] >= pivotvlue and rightmark >= leftmark:
            rightmark -= 1

        if leftmark > rightmark:
            done = True
        else:
            alist[leftmark], alist[rightmark] = alist[rightmark], alist[leftmark]
    alist[rightmark], alist[first] = alist[first], alist[rightmark]
    return rightmark

def qsort(lists, low, high):
    print('this', lists[low:high + 1])
    left = low
    right = high
    pivot = low
    while high > low:
        while lists[high] >= lists[pivot] and high > low:
            high -= 1
        while lists[low] <= lists[pivot] and high > low:
            low += 1
        lists[high], lists[low] = lists[low], lists[high]
        print('swape', lists, high, low)
        high -= 1
        low += 1
    pivot = low - 1
    lists[left], lists[pivot] = lists[pivot], lists[left]
    print('parted', lists, 'left', left, 'pivot', pivot)
    if pivot > left + 1:
        qsort(lists, left, pivot - 1)
    if pivot < right - 1:
        qsort(lists, pivot + 1, right)
    return lists

alist = [54, 26, 93, 17, 77, 31, 44, 55, 20]
alist2 = [1]
print(alist2)
quickSort(alist2)
print(alist2)
print(alist)
quickSort(alist)
print(alist)


# encoding: utf-8
# ---------------------------------------------------------------------------
#  Copyright (C) 2008-2014, IPython Development Team and Enthought, Inc.
#  Distributed under the terms of the BSD License.  See COPYING.rst.
# ---------------------------------------------------------------------------

import unittest
from random import randrange

from distarray.externals.six.moves import range

from distarray.testing import ContextTestCase
from distarray.dist import maps as client_map
from distarray.dist.maps import MapBase

class TestClientMap(ContextTestCase):

    def test_2D_bn(self):
        nrows, ncols = 31, 53
        cm = client_map.Distribution.from_shape(self.context,
                (nrows, ncols),
                {0: 'b'},
                (4, 1))
        chunksize = (nrows // 4) + 1
        for _ in range(100):
            r, c = randrange(nrows), randrange(ncols)
            rank = r // chunksize
            self.assertSequenceEqual(cm.owning_ranks((r,c)), [rank])

    def test_2D_bb(self):
        nrows, ncols = 3, 5
        nprocs_per_dim = 2
        cm = client_map.Distribution.from_shape(
                self.context, (nrows, ncols), ('b', 'b'),
                (nprocs_per_dim, nprocs_per_dim))
        row_chunks = nrows // nprocs_per_dim + 1
        col_chunks = ncols // nprocs_per_dim + 1
        for r in range(nrows):
            for c in range(ncols):
                rank = (r // row_chunks) * nprocs_per_dim + (c // col_chunks)
                actual = cm.owning_ranks((r,c))
                self.assertSequenceEqual(actual, [rank])

    def test_2D_cc(self):
        nrows, ncols = 3, 5
        nprocs_per_dim = 2
        cm = client_map.Distribution.from_shape(
                self.context, (nrows, ncols), ('c', 'c'),
                (nprocs_per_dim, nprocs_per_dim))
        for r in range(nrows):
            for c in range(ncols):
                rank = (r % nprocs_per_dim) * nprocs_per_dim + (c % nprocs_per_dim)
                actual = cm.owning_ranks((r,c))
                self.assertSequenceEqual(actual, [rank])

    def test_is_compatible(self):
        nr, nc, nd = 10**5, 10**6, 10**4

        cm0 = client_map.Distribution.from_shape(
                 self.context, (nr, nc, nd), ('b', 'c', 'n'))
        self.assertTrue(cm0.is_compatible(cm0))

        cm1 = client_map.Distribution.from_shape(
                 self.context, (nr, nc, nd), ('b', 'c', 'n'))
        self.assertTrue(cm1.is_compatible(cm1))

        self.assertTrue(cm0.is_compatible(cm1))
        self.assertTrue(cm1.is_compatible(cm0))

        nr -= 1; nc -= 1; nd -= 1

        cm2 = client_map.Distribution.from_shape(
                 self.context, (nr, nc, nd), ('b', 'c', 'n'))

        self.assertFalse(cm1.is_compatible(cm2))
        self.assertFalse(cm2.is_compatible(cm1))

    def test_reduce(self):
        nr, nc, nd = 10**5, 10**6, 10**4

        dist = client_map.Distribution.from_shape(
                 self.context, (nr, nc, nd), ('b', 'c', 'n'),
                 grid_shape=(2, 2, 1))

        new_dist0 = dist.reduce(axes=[0])
        self.assertEqual(new_dist0.dist, ('c', 'n'))
        self.assertSequenceEqual(new_dist0.shape, (nc, nd))
        self.assertEqual(new_dist0.grid_shape, dist.grid_shape[1:])
        self.assertLess(set(new_dist0.targets), set(dist.targets))

        new_dist1 = dist.reduce(axes=[1])
        self.assertEqual(new_dist1.dist, ('b', 'n'))
        self.assertSequenceEqual(new_dist1.shape, (nr, nd))
        self.assertEqual(new_dist1.grid_shape, dist.grid_shape[:1]+dist.grid_shape[2:])
        self.assertLess(set(new_dist1.targets), set(dist.targets))

        new_dist2 = dist.reduce(axes=[2])
        self.assertEqual(new_dist2.dist, ('b', 'c'))
        self.assertSequenceEqual(new_dist2.shape, (nr, nc))
        self.assertEqual(new_dist2.grid_shape, dist.grid_shape[:-1])
        self.assertEqual(set(new_dist2.targets), set(dist.targets))

    def test_reduce_0D(self):
        N = 10**5
        dist = client_map.Distribution.from_shape(self.context, (N,))
        new_dist = dist.reduce(axes=[0])
        self.assertEqual(new_dist.dist, ())
        self.assertSequenceEqual(new_dist.shape, ())
        self.assertEqual(new_dist.grid_shape, ())
        self.assertEqual(set(new_dist.targets), set(dist.targets[:1]))

class TestDunderMethods(ContextTestCase):

    @classmethod
    def setUpClass(cls):
        super(TestDunderMethods, cls).setUpClass()
        cls.shape = (3, 4, 5, 6)
        cls.cm = client_map.Distribution.from_shape(cls.context, cls.shape)

    def test___len__(self):
        self.assertEqual(len(self.cm), 4)

    def test___getitem__(self):
        for m in self.cm:
            self.assertTrue(isinstance(m, MapBase))

        self.assertEqual(self.cm[0].dist, 'b')
        self.assertEqual(self.cm[1].dist, 'n')
        self.assertEqual(self.cm[2].dist, 'n')
        self.assertEqual(self.cm[-1].dist, 'n')

class TestDistributionCreation(ContextTestCase):
    def test_all_n_dist(self):
        distribution = client_map.Distribution.from_shape(self.context,
                                                          shape=(3, 3),
                                                          dist=('n', 'n'))
        self.context.ones(distribution)

if __name__ == '__main__':
    unittest.main(verbosity=2)


#!/usr/bin/env python
import os
import re
import warnings

import selenosis

# A dict for mapping test case classes to their import paths, to allow passing
# TestCaseClass.test_function as shorthand to runtests.py
TEST_CASE_MODULE_PATHS = {
    'TestAdminWidgets': 'nested_admin.tests.admin_widgets.tests',
    'TestWidgetMediaOrder': 'nested_admin.tests.admin_widgets.tests',
    'TestGenericInlineAdmin': 'nested_admin.tests.gfk.tests',
    'VisualComparisonTestCase': 'nested_admin.tests.one_deep.tests',
    'TestDeepNesting': 'nested_admin.tests.three_deep.tests',
    'TestStackedInlineAdmin': 'nested_admin.tests.two_deep.tests',
    'TestTabularInlineAdmin': 'nested_admin.tests.two_deep.tests',
    'TestSortablesWithExtra': 'nested_admin.tests.two_deep.tests',
    'TestIdenticalPrefixes': 'nested_admin.tests.identical_prefixes.tests',
}

def expand_test_module(module):
    module = os.path.normpath(module)
    matches = re.search(r'^([^/.]+)(\.[^./]+)?$', module)
    if not matches:
        return module
    cls, test_fn = matches.groups()
    if not test_fn:
        test_fn = ''
    if cls not in TEST_CASE_MODULE_PATHS:
        return module
    return "%s.%s%s" % (TEST_CASE_MODULE_PATHS[cls], cls, test_fn)

class RunTests(selenosis.RunTests):

    def execute(self, flags, test_labels):
        test_labels = [expand_test_module(m) for m in test_labels]
        super(RunTests, self).execute(flags, test_labels)

def main():
    warnings.simplefilter("error", Warning)
    warnings.filterwarnings(
        "ignore",
        "name used for saved screenshot does not match file type",
        UserWarning)
    runtests = RunTests("nested_admin.tests.settings", "nested_admin")
    runtests()

if __name__ == '__main__':
    main()


# класс точка в нмерном пространстве, функции растояния между точками,
# середины отрезка, если н у точек разное, ошибкаьзначения
from math import sqrt

class Dot (object):
    def __init__(self, *koord):
        self.n = len(koord)+1
        self.koord = koord

    def __str__(self):
        s = ''
        for k in self.koord:
            s += str(k)+', '
        s = s[:-2]
        return s

    def distance(self, sobject):
        if self.n != sobject.n:
            raise ValueError
        else:
            v = 0
            for i in range(len(self.koord)):
                v += (sobject.koord[i] - self.koord[i])**2
            v = sqrt(v)
            return v

    def middle(self, sobject):
        k = []
        for i in range(len(self.koord)):
            k.append((sobject.koord[i]+self.koord[i])/2)
        return Dot(*k)

for A, B in (Dot(1, 2, 3), Dot(3, 4, 5)), (Dot(1, 2), Dot(3)):
    print(A)
    try:
        print("{:.3f}".format(A.distance(B)))
        print(A.middle(B))
    except ValueError:
        print("ERROR")