import os
import src.common.common as cmn
from src.snowflake.snowflake_client import snowflake_client as sf_client
from src.deployer.deployer import deployer
from src.deploy_logger.logger import deploy_logger
from src.wrangler.wrangler import wrangler
#from yaml_writer.yaml_writer import yaml_writer
#from yaml_reader.yaml_reader import yaml_reader
from src.common.common import *
from src.hasher.hasher import hasher
import logging
import shutil
from time import sleep
import time
#from random import random
import threading
import random
import hashlib
import re
import yaml
from pathlib import Path
from jinja2 import Environment, FileSystemLoader, meta, StrictUndefined
from src.common.exceptions import object_type_not_supported
from src.common.enums import HANDLE_OWNERSHIP_OPTION
import traceback
from src.validator.validator import validator
from src.configurator.configurator import configurator

def role(ref_val: str) -> str:
    val = validator()
    val.validate_role_exists('snowflake', ref_val, template_path)

    parts = ref_val.split('__')
    if len(parts) == 1:
        #ref_id = ref_val
        jinja_ref_id = 'role__' + ref_val  # obj_type defined in loop
        processed_name = get_sf_role_name(ref_val, config['ENV_ROLE_PREFIX'])
    else:
        raise Exception('Must use ref (not role()) for reference ' + ref_val + ' in file ' + template_path)

    r={}
    if file_ext.upper() in ('YML','YAML'):
        r['path'] = template_path
        r['code_path'] = ''
    else:
        raise Exception('Invalid file type in ' + template_path)

    #r['path'] = template_path # defined in the template loop that render (and hence ref) is called from
    r['obj_name'] = obj_name  # defined in the template loop that render (and hence ref) is called from
    r['ref_id'] = ref_id # defined in the tempalte loop that rendered (and hence ref) is called from
    r['obj_type'] = obj_type  # defined in the template loop that render (and hence ref) is called from
    r['deps'] = [jinja_ref_id]
    r['sort_order'] = 1
    _references.append(r)

    return processed_name

def warehouse(ref_val: str) -> str:
    val = validator()
    val.validate_warehouse_exists('snowflake', ref_val, template_path)

    parts = ref_val.split('__')
    if len(parts) == 1:
        #ref_id = ref_val
        jinja_ref_id = 'warehouse__' + ref_val  # obj_type defined in loop
        processed_name = get_sf_warehouse_name(ref_val, config['ENV_WAREHOUSE_PREFIX'])
    else:
        raise Exception('Must use ref (not warehouse()) for reference ' + ref_val + ' in file ' + template_path)

    r={}
    if file_ext.upper() in ('YML','YAML'):
        r['path'] = template_path
        r['code_path'] = ''
    else:
        raise Exception('Invalid file type in ' + template_path)

    #r['path'] = template_path # defined in the template loop that render (and hence ref) is called from
    r['obj_name'] = obj_name  # defined in the template loop that render (and hence ref) is called from
    r['ref_id'] = ref_id # defined in the tempalte loop that rendered (and hence ref) is called from
    r['obj_type'] = obj_type  # defined in the template loop that render (and hence ref) is called from
    r['deps'] = [jinja_ref_id]
    r['sort_order'] = 1
    _references.append(r)

    return processed_name

def ref(ref_val: str)->str:

    val = validator()
    val.validate_ref_exists('snowflake', ref_val, template_path)

    parts = ref_val.split('__')
    if len(parts) == 1:
        raise Exception('Must use account level function (warehouse() or role()) for ref ' + ref_val + ' in file ' + template_path)
    elif len(parts) == 3:
        db = parts[0]
        schema = parts[1]
        object = parts[2]
        jinja_ref_id = db + '.' + schema + '.' + object
        db_env = get_sf_database_name(parts[0], config['ENV_DATABASE_PREFIX'])
        processed_name = db_env + '.' + schema + '.' + object
    else:
        raise Exception('Invalid formatted ref value in ' + template_path)

    #schema_id = db + '.' + schema
    r={}
    if file_ext.upper() in ('YML','YAML'):
        r['path'] = template_path
        r['code_path'] = ''
    else:
        yml_obj_path = re.sub(file_ext+'$','yml',template_path)
        #yaml_obj_path = re.sub(file_ext+'$','yaml',template_path)
        r['path'] = yml_obj_path
        r['code_path'] = template_path
    #r['path'] = template_path # defined in the template loop that render (and hence ref) is called from
    r['obj_name'] = obj_name  # defined in the template loop that render (and hence ref) is called from
    r['ref_id'] = ref_id # defined in the tempalte loop that rendered (and hence ref) is called from
    r['obj_type'] = obj_type  # defined in the template loop that render (and hence ref) is called from
    r['deps'] = [jinja_ref_id]
    r['sort_order'] = sort_order
    _references.append(r)

    return processed_name

def task(semaphore, tn:str, ref_id:str, data:dict, completed:list, processing:list, deploy, base_rendered_path:str, deploy_config:dict, logger, available_roles:list, val:validator, object_state_dict, db_hash_dict):
    thread_start_time = time.time() # time is needed in case of an error (but really re-set the time after the semaphore starts the thread)
    #object_name:str
    try:
        # Connect with the semaphore so particular thread knows whether it is allowed to process
        # The semaphore is like a "thread pool" that controls the number of concurrent threads processing at once
        with semaphore:
            thread_start_time = time.time()
            if ref_id not in processing:
                object_name = data[ref_id]['obj_name']
                processing.append(ref_id)
                name = threading.current_thread().name
                good_to_process = True
                for d in data[ref_id]['deps']:
                    if d not in completed:
                        good_to_process = False # if any deps has not been completed, then need to skip (means there's multiple next references and this thread will fire again after that reference)

                #if not good_to_process:
                    #print('Skipping thread ' + tn + ' as not all dependencies have finished')
                #else:
                if good_to_process:
                    #print(f'{object_name} - Starting')
                    logger.log(object_name,'Start')
                    # DO THE WORK HERE SINCE GOOD TO PROCESS
                    #print('Processing thread ' + tn)
                    # Wait a random number of seconds (between 2-5) to demonstrate threads running vs waiting to process
                    #value = random.randint(2, 5)
                    #sleep(value)

                    # Read in YAML config
                    #print('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
                    #print(data[object_name])
                    config_path = base_rendered_path + data[ref_id]['path']
                    with open(config_path, "r") as yamlfile:
                        config_raw = yaml.load(yamlfile, Loader=yaml.FullLoader)

                    if data[ref_id]['obj_type'] in ('procedure','function','masking_policy','row_access_policy','task'):
                        #if data[ref_id]['obj_type'] == 'row_access_policy':
                        #    #print('#################################')
                        #    #print(data[ref_id])
                        #    #print('#################################')
                        code_path = base_rendered_path + data[ref_id]['code_path']
                        with open(code_path, "r") as codefile:
                            body_code = codefile.read()

                        file_hash_code = hashlib.sha256(open(code_path,'rb').read()).hexdigest()
                    else:
                        file_hash_code = None

                    file_hash = hashlib.sha256(open(config_path,'rb').read()).hexdigest()

                    #deploy_db_name = deploy_config['DEPLOY_DATABASE_NAME']
                    #deploy_role = deploy_config['DEPLOY_ROLE']
                    #handle_ownership = deploy_config['HANDLE_OWNERSHIP']
                    #deploy_env = deploy_config['DEPLOY_ENV']

                    #print('######## db_hash_dict ##########')
                    #print(db_hash_dict)
                    #print('################################')
                    object_type = data[ref_id]['obj_type']
                    # Deploy
                    if object_type == 'database':
                        val.validate_database(config_raw)
                        return_status = deploy.deploy_database(object_name, file_hash, config_raw, object_state_dict, db_hash_dict['database'])
                    elif object_type == 'schema':
                        val.validate_schema(config_raw)
                        return_status = deploy.deploy_schema(object_name, file_hash, config_raw, object_state_dict, db_hash_dict['schema'])
                    elif object_type == 'tag':
                        val.validate_tag(config_raw)
                        return_status = deploy.deploy_tag(object_name, file_hash, config_raw, object_state_dict, db_hash_dict['tag'])
                    elif object_type == 'object':
                        val.validate_object(config_raw)
                        return_status = deploy.deploy_object(object_name, file_hash, config_raw, object_state_dict, db_hash_dict['object'])
                    elif object_type == 'warehouse':
                        val.validate_warehouse(config_raw)
                        return_status = deploy.deploy_warehouse(object_name, file_hash, config_raw, object_state_dict, db_hash_dict['warehouse'])
                    elif object_type == 'role':
                        val.validate_role(config_raw)
                        return_status = deploy.deploy_role(object_name, file_hash, config_raw, object_state_dict, db_hash_dict['role'])
                    elif object_type == 'procedure':
                        val.validate_procedure(config_raw)
                        return_status = deploy.deploy_procedure(object_name, file_hash, file_hash_code, config_raw, body_code, object_state_dict, db_hash_dict['procedure'])
                    elif object_type == 'function':
                        val.validate_function(config_raw)
                        return_status = deploy.deploy_function(object_name, file_hash, file_hash_code, config_raw, body_code, object_state_dict, db_hash_dict['function'])
                    elif object_type == 'task':
                        val.validate_task(config_raw)
                        return_status = deploy.deploy_task(object_name, file_hash, file_hash_code, config_raw, body_code, object_state_dict, db_hash_dict['task'])
                    elif object_type == 'masking_policy':
                        val.validate_masking_policy(config_raw)
                        return_status = deploy.deploy_masking_policy(object_name, file_hash, file_hash_code, config_raw, body_code, object_state_dict, db_hash_dict['masking_policy'])
                        #dummy = True
                    elif object_type == 'row_access_policy':
                        val.validate_row_access_policy(config_raw)
                        return_status = deploy.deploy_row_access_policy(object_name, file_hash, file_hash_code, config_raw, body_code, object_state_dict, db_hash_dict['row_access_policy'])
                        #dummy = True
                    else:
                        raise object_type_not_supported(object_type)
                    #print(f'Thread {name} Mid')
                    completed.append(ref_id)

                    #filter = [d for d in data if d['num'] == next]
                    #threads = []
                    for next_ref_id in data[ref_id]['next']:
                        tn2 = 'thread_' + next_ref_id
                        t2 = threading.Thread(target=task, name=tn2, args=(semaphore, tn2, next_ref_id, data, completed, processing, deploy, base_rendered_path, deploy_config, logger, available_roles, val, object_state_dict, db_hash_dict))
                        #                                                   semaphore, tn, object_name, data, completed
                        t2.start()

                    if return_status == 'C':
                        msg = 'created (%s seconds)' % (round(time.time() - thread_start_time,1))
                        logger.log(object_name,msg)
                        #print(f'{object_name} - created (%s seconds)' % (round(time.time() - thread_start_time,1)))
                    elif return_status == 'U':
                        msg = 'updated (%s seconds)' % (round(time.time() - thread_start_time,1))
                        logger.log(object_name,msg)
                        #print(f'{object_name} - updated (%s seconds)' % (round(time.time() - thread_start_time,1)))
                    elif return_status == 'I':
                        msg = 'ignored (%s seconds)' % (round(time.time() - thread_start_time,1))
                        logger.log(object_name, msg)
                    elif return_status == 'E':
                        msg = 'ignored - not deployable in this env (%s seconds)' % (round(time.time() - thread_start_time,1))
                        logger.log(object_name, msg)
                        #print(f'{object_name} - ignored - nothing to update (%s seconds)' % (round(time.time() - thread_start_time,1)))
                    #print(f'{object_name} - Completed in %s seconds' % (round(time.time() - thread_start_time,1)))

                processing.remove(ref_id)
    except Exception as ex:
        #print(str(ex))
        traceback_text = traceback.format_exc() # get error traceback
        msg = 'error (%s seconds) - see error log at end' % (round(time.time() - thread_start_time,1))
        logger.log(object_name,msg)
        logger.log_error(str(ex), object_name, traceback_text)
        #raise

def check_for_circular_ref(map, key, original_key):
    #print('key:' + key + '; next:'+str(map[key]['next']))
    if map[key]['next'] == []:
        is_circle = False
        #print('key:' + key + '; is_circle:'+str(is_circle))
    else:
        for n in map[key]['next']:
            if n == original_key:
                is_circle = True
            else:
                is_circle = check_for_circular_ref(map, n, original_key) # check the next of the next
                break
            #print('key:' + key + '; n:'+n+ '; is_circle:'+str(is_circle))
    return is_circle

def snowflake_deploy(args:dict):
    global template_path
    global file_ext
    global config
    global obj_name
    global ref_id
    global obj_type
    global sort_order
    global _references

    #print('this is the deploy file')
    start_time = time.time()

    logging.basicConfig(level=logging.WARNING)
    logger = deploy_logger('info')

    val = validator()
    val.validate_directory_structure('snowflake')

    # Check if data directory exists
    snowflake_directory_exists = os.path.exists('snowflake/')
    if not snowflake_directory_exists:
        raise Exception('Snowflake directory does not exist.  See documentation.')

    conf = configurator(args)
    config = conf.get_config()

    ###############################################################################################################
    #                                                   Defaults
    ###############################################################################################################

    deploy_db_name = config['DEPLOY_DATABASE_NAME']
    deploy_role = config['DEPLOY_ROLE']
    handle_ownership = config['HANDLE_OWNERSHIP']
    deploy_env = config['DEPLOY_ENV']
    excluded_databases = config['EXCLUDED_DATABASES']
    import_databases = config['IMPORT_DATABASES']

    try:
        ###############################################################################################################
        #                                                   Connect to Snowflake
        ###############################################################################################################
        logger.log('','Connecting to Snowflake ...')
        sf = sf_client(config['SNOWFLAKE_PRIVATE_KEY'], config['SNOWFLAKE_PRIVATE_KEY_PASSWORD'], config['SNOWFLAKE_ACCOUNT'], config['SNOWFLAKE_USERNAME'], config['SNOWFLAKE_WAREHOUSE'], config['database'], config['schema'] )
        #print('Connected to Snowflake')
        logger.log('','Connected to Snowflake!')
        logger.log('','Getting things set up')

        # Get current role
        available_roles = []
        current_role = sf.current_role_get()
        available_roles.append(current_role)
        #ignore_roles_list = config['STANDARD_ROLES']
        #ignore_roles_list.append(current_role) # ignore out of the box roles PLUS the role that owns deployments (which can be hard coded in the files)

        #####################################################
        # Install Deploy DB if not exists

        hsh = hasher()
        wrangle = wrangler(sf)
        deploy = deployer(sf, deploy_db_name, deploy_role, handle_ownership, available_roles, deploy_env, hsh)
        deploy.check_and_install_deployer_db()

        # Get Get hash values
        logger.log('','Getting current state Snowflake log')
        object_state_dict = sf.deploy_db_object_state_get(deploy_db_name, deploy_env)

        # Set up jinja environment and load all data files
        _references = []
        env = Environment(loader=FileSystemLoader(['snowflake']), undefined=StrictUndefined)
        # register the python function for jinja environment
        env.globals['ref'] = ref
        env.globals['warehouse'] = warehouse
        env.globals['role'] = role

        base_rendered_path = 'metaops-deploy-rendered/'

        logger.log('','Loading config files')

        # Delete existing rendered
        dirpath = Path(base_rendered_path)
        if dirpath.exists() and dirpath.is_dir():
            #os.remove(base_rendered_path)
            shutil.rmtree(base_rendered_path)

        # Loop files and render
        for template_path in env.list_templates():
            template = env.get_template(template_path)
            #print(template_path)
            filename = os.path.splitext(os.path.basename(template_path))[0]
            folders = template_path.split('/')
            top_level = folders[0]

            if folders[0].upper() == 'ACCOUNT':
                if folders[1].upper() == 'WAREHOUSES':
                    obj_type = 'warehouse'
                    raw_name = folders[2].split('.')[0]
                    obj_name = get_sf_warehouse_name(raw_name, config['ENV_WAREHOUSE_PREFIX'])
                    ref_id = obj_type + '__' + raw_name
                    file_ext = folders[2].split('.')[1]
                    deps = []
                    sort_order = 1
                elif folders[1].upper() == 'ROLES':
                    obj_type = 'role'
                    raw_name = folders[2].split('.')[0]
                    obj_name = get_sf_role_name(raw_name, config['ENV_ROLE_PREFIX'])
                    ref_id = obj_type + '__' + raw_name
                    file_ext = folders[2].split('.')[1]
                    deps = []
                    sort_order = 1
                else:
                    raise Exception('Invalid folder structure - folder not valid:' + folders[0] + '/' + folders[1])
            elif folders[0].upper() == 'DATA':  # data
                db_raw = folders[1]
                db_env = get_sf_database_name(db_raw, config['ENV_DATABASE_PREFIX'])
                if( len(folders) == 3 ):
                    if( folders[2].split('.')[0].upper() != 'DATABASE' ):
                        raise Exception('database ' + folders[1] + ' does not contain database.yml config')
                    obj_type = 'database'
                    obj_name = db_env
                    ref_id = db_raw
                    file_ext = folders[2].split('.')[1]
                    deps = []
                    sort_order = 2
                elif( len(folders) == 4):
                    if( folders[3].split('.')[0].upper() != 'SCHEMA' ):
                        raise Exception('schema ' + folders[1] + '.' + folders[2] + ' does not contain schema.yml config')
                    obj_type = 'schema'
                    raw_schema = folders[2]
                    obj_name = db_env + '.' + raw_schema
                    ref_id = db_raw + '.' + raw_schema
                    file_ext = folders[3].split('.')[1]
                    deps = [db_raw]
                    sort_order = 3
                elif( len(folders) == 5):
                    #if( folders[2].upper() not in ('TAGS') ):
                    if folders[3].upper() == 'TAGS':
                        obj_type = 'tag'
                        raw_name = folders[4].split('.')[0].split('__')[2]
                        obj_name = db_env + '.' + folders[2] + '.' + raw_name
                        ref_id = db_raw + '.' + folders[2] + '.' + raw_name
                        schema_id = db_raw + '.' + folders[2]
                        file_ext = folders[4].split('.')[1]
                        deps = [schema_id]
                        sort_order = 4
                    elif folders[3].upper() == 'OBJECTS':
                        obj_type = 'object'
                        raw_name = folders[4].split('.')[0].split('__')[2]
                        obj_name = db_env + '.' + folders[2] + '.' + raw_name
                        ref_id = db_raw + '.' + folders[2] + '.' + raw_name
                        schema_id = db_raw + '.' + folders[2]
                        file_ext = folders[4].split('.')[1]
                        deps = [schema_id]
                        sort_order = 4
                    elif folders[3].upper() == 'PROCEDURES':
                        obj_type = 'procedure'
                        raw_name = folders[4].split('.')[0].split('__')[2] + '__' + folders[4].split('.')[0].split('__')[3]
                        obj_name = db_env + '.' + folders[2] + '.' + raw_name
                        ref_id = db_raw + '.' + folders[2] + '.' + raw_name
                        schema_id = db_raw + '.' + folders[2]
                        file_ext = folders[4].split('.')[1]
                        deps = [schema_id]
                        sort_order = 4
                    elif folders[3].upper() == 'FUNCTIONS':
                        obj_type = 'function'
                        raw_name = folders[4].split('.')[0].split('__')[2] + '__' + folders[4].split('.')[0].split('__')[3]
                        obj_name = db_env + '.' + folders[2] + '.' + raw_name
                        ref_id = db_raw + '.' + folders[2] + '.' + raw_name
                        schema_id = db_raw + '.' + folders[2]
                        file_ext = folders[4].split('.')[1]
                        deps = [schema_id]
                        sort_order = 4
                    elif folders[3].upper() == 'TASKS':
                        obj_type = 'task'
                        raw_name = folders[4].split('.')[0].split('__')[2]
                        obj_name = db_env + '.' + folders[2] + '.' + raw_name
                        ref_id = db_raw + '.' + folders[2] + '.' + raw_name
                        schema_id = db_raw + '.' + folders[2]
                        file_ext = folders[4].split('.')[1]
                        deps = [schema_id]
                        sort_order = 4
                    elif folders[3].upper() == 'MASKING_POLICIES':
                        obj_type = 'masking_policy'
                        raw_name = folders[4].split('.')[0].split('__')[2]
                        obj_name = db_env + '.' + folders[2] + '.' + raw_name
                        ref_id = db_raw + '.' + folders[2] + '.' + raw_name
                        schema_id = db_raw + '.' + folders[2]
                        file_ext = folders[4].split('.')[1]
                        deps = [schema_id]
                        sort_order = 4
                    elif folders[3].upper() == 'ROW_ACCESS_POLICIES':
                        obj_type = 'row_access_policy'
                        raw_name = folders[4].split('.')[0].split('__')[2]
                        obj_name = db_env + '.' + folders[2] + '.' + raw_name
                        ref_id = db_raw + '.' + folders[2] + '.' + raw_name
                        schema_id = db_raw + '.' + folders[2]
                        file_ext = folders[4].split('.')[1]
                        deps = [schema_id]
                        sort_order = 4
                    else:
                        raise object_type_not_supported(folders[3])
                        #raise Exception('object type ' + folders[2] + ' not currently support')

            else:
                raise Exception('Invalid folder structure - top folder not valid:' + folders[0])

            r = {}
            if file_ext.upper() in ('YML','YAML'):
                r['path'] = template_path
                r['code_path'] = ''
            else:
                yml_obj_path = re.sub(file_ext+'$','yml',template_path)
                #print(yml_obj_path)
                #yaml_obj_path = re.sub(file_ext+'$','yaml',template_path)
                r['path'] = yml_obj_path
                r['code_path'] = template_path
            #print('###########################################')
            r['obj_name'] = obj_name
            r['ref_id'] = ref_id
            r['obj_type'] = obj_type
            r['deps'] = deps
            r['sort_order'] = sort_order
            #print(config['VARS'])

            _references.append(r)
            new_path = base_rendered_path + template_path
            output_file = Path(new_path)
            output_file.parent.mkdir(exist_ok=True, parents=True)
            with open(new_path, "w") as f:
                f.write(template.render(config['VARS']))

        #print('###################################')
        #print(_references)
        #print('###################################')
        #####################################################
        # Translate References into a map of downstream and upstream for each obj
        map = {}
        for r in _references:
            path = r['path']
            code_path = r['code_path']
            obj_name = r['obj_name']
            ref_id = r['ref_id']
            obj_type = r['obj_type']
            deps = r['deps']
            sort_order = r['sort_order']

            # Add object to path
            if ref_id not in map.keys():
                map[ref_id] = {}
                map[ref_id]['path'] = path
                map[ref_id]['code_path'] = code_path
                map[ref_id]['obj_type'] = obj_type
                map[ref_id]['obj_name'] = obj_name
                map[ref_id]['deps'] = []
                map[ref_id]['next'] = []
                map[ref_id]['sort_order'] = sort_order
            else:
                map[ref_id]['path'] = path # if added by Nexts, need to update path of object
                map[ref_id]['obj_type'] = obj_type
                map[ref_id]['obj_name'] = obj_name
                map[ref_id]['sort_order'] = sort_order
                if (code_path is not None and code_path != '' and 'code_path' in map[ref_id]) and (map[ref_id]['code_path'] is None or map[ref_id]['code_path'] == ''):
                    map[ref_id]['code_path'] = code_path
                elif code_path is not None and code_path != '' and 'code_path' not in map[ref_id]:
                    map[ref_id]['code_path'] = code_path

            for d in deps:
                # Deps -
                if d not in map[ref_id]['deps']:
                    map[ref_id]['deps'].append(d)

                # Nexts
                if d not in map.keys():
                    map[d] = {}
                    map[d]['deps'] = []
                    map[d]['next'] = []
                if ref_id not in map[d]['next']:
                    map[d]['next'].append(ref_id)

        #{
        #'CONTROL.GOVERNANCE.ENV':
        #    { 'deps': ['CONTROL.GOVERNANCE'], 'next': ['CONTROL','CONTROL2']}
        #
        #, 'CONTROL.GOVERNANCE':
        #    {'deps': ['CONTROL'], 'next': ['CONTROL.GOVERNANCE.ENV']}
        #
        #, 'CONTROL':
        #    {'deps': ['CONTROL.GOVERNANCE.ENV'], 'next': ['CONTROL.GOVERNANCE']}
        #}

        #####################################################
        # Check for circular dependencies

        circular_keys = []
        for key in map.keys():
            is_circle = check_for_circular_ref(map, key, key)
            if is_circle and key not in circular_keys:
                circular_keys.append(key)
        #print('******** CIRCULAR KEYS ********')
        #print(circular_keys)
        #print('*******************************')
        if circular_keys != []:
            circular_keys_string = '[' + ', '.join(circular_keys) + ']'
            raise Exception('Circular keys found in the following objects: ' + circular_keys_string)

        #####################################################
        # Get starting objects (ones with no dependencies)
        #print('&&&&&&&&&&&&&&&&&&&&&&&&&&&')
        #print(map)
        #print('&&&&&&&&&&&&&&&&&&&&&&&&&&&')
        map_start = []
        for key in map.keys():
            if map[key]['deps'] == []:
                map_start.append(key)
        #print(map_start)

        # Semaphore sets the number of concurrent threads running at any one time
        semaphore = threading.Semaphore(config['MAX_THREADS'])

        #####################################################
        # Get Object Types to get current types

        map_object_types = []
        for key in map.keys():
            obj_type = map[key]['obj_type']
            if obj_type not in map_object_types:
                map_object_types.append(obj_type)

        #map_object_types_unsorted = []
        #object_types_unique = []
        #for key in map.keys():
        #    k = {}
        #    obj_type = map[key]['obj_type']
        #    sort_order = map[key]['sort_order']
        #    if obj_type not in object_types_unique:
        #        k['object_type'] = obj_type
        #        k['sort_order'] = sort_order
        #        map_object_types_unsorted.append(k)
        #        object_types_unique.append(obj_type)

        #map_object_types = sorted(map_object_types_unsorted, key=lambda d: d['sort_order'])

        ignore_roles_list = config['STANDARD_ROLES']
        ignore_roles_list.append(current_role) # ignore out of the box roles PLUS the role that owns deployments (which can be hard coded in the files)

        #print('------------------')
        #print(map_object_types)
        #print('------------------')

        db_hash_dict = {}
        dbs = []
        schemas = []
        tags = []
        objects = []
        procedures = []
        functions = []
        tasks = []
        masking_policies = []
        row_access_policies = []
        if 'warehouse' in map_object_types:
            logger.log('','Getting current Snowflake WAREHOUSES to calculate diffs from config')
            whs = wrangle.wrangle_warehouse(config['ENV_WAREHOUSE_PREFIX'], config['ENV_DATABASE_PREFIX'], config['ENV_ROLE_PREFIX'], config['DEPLOY_DATABASE_NAME'], ignore_roles_list, config['DEPLOY_TAGS'],config['DEPLOY_ROLE'], available_roles, config['HANDLE_OWNERSHIP'],semaphore)
            db_hash_dict['warehouse'] = hsh.hash_warehouse_all(whs)

        if 'role' in map_object_types:
            logger.log('','Getting current Snowflake ROLES to calculate diffs from config')
            rs = wrangle.wrangle_role(config['ENV_ROLE_PREFIX'], config['ENV_DATABASE_PREFIX'], config['DEPLOY_DATABASE_NAME'], ignore_roles_list, config['DEPLOY_TAGS'],config['DEPLOY_ROLE'], available_roles, config['HANDLE_OWNERSHIP'],semaphore)
            db_hash_dict['role'] = hsh.hash_role_all(rs)

        if 'database' in map_object_types:
            logger.log('','Getting current Snowflake DATABASES to calculate diffs from config')
            dbs = wrangle.wrangle_database(config['ENV_DATABASE_PREFIX'], config['ENV_ROLE_PREFIX'], excluded_databases, config['DEPLOY_DATABASE_NAME'], ignore_roles_list, config['DEPLOY_TAGS'],config['DEPLOY_ROLE'], available_roles, config['HANDLE_OWNERSHIP'], import_databases,semaphore)
            db_hash_dict['database'] = hsh.hash_database_all(dbs)

        if 'schema' in map_object_types:
            logger.log('','Getting current Snowflake SCHEMAS to calculate diffs from config')
            for db in dbs:
                schemas_new = wrangle.wrangle_schema(db['DATABASE_NAME'], config['ENV_DATABASE_PREFIX'], config['ENV_ROLE_PREFIX'], config['DEPLOY_DATABASE_NAME'], ignore_roles_list, config['DEPLOY_TAGS'],config['DEPLOY_ROLE'], available_roles, config['HANDLE_OWNERSHIP'],semaphore)
                schemas = schemas + schemas_new
            db_hash_dict['schema'] = hsh.hash_schema_all(schemas)

        if 'tag' in map_object_types:
            logger.log('','Getting current Snowflake TAGS to calculate diffs from config')
            for schema in schemas:
                tags_new = wrangle.wrangle_tag(schema['DATABASE_NAME'], schema['SCHEMA_NAME'], config['ENV_DATABASE_PREFIX'], config['DEPLOY_DATABASE_NAME'], ignore_roles_list, config['DEPLOY_TAGS'],config['DEPLOY_ROLE'], available_roles, config['HANDLE_OWNERSHIP'],semaphore)
                tags = tags + tags_new
            db_hash_dict['tag'] = hsh.hash_tag_all(tags)

        if 'object' in map_object_types:
            logger.log('','Getting current Snowflake OBJECTS to calculate diffs from config')
            for schema in schemas:
                objects_new = wrangle.wrangle_object(schema['DATABASE_NAME'], schema['SCHEMA_NAME'], config['ENV_DATABASE_PREFIX'], config['ENV_ROLE_PREFIX'], config['DEPLOY_DATABASE_NAME'], ignore_roles_list, config['DEPLOY_TAGS'],config['DEPLOY_ROLE'], available_roles, config['HANDLE_OWNERSHIP'],semaphore)
                objects = objects + objects_new
            db_hash_dict['object'] = hsh.hash_object_all(objects)
            #print('^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^')
            #print(db_hash_dict['object'])
            #print('~~~~~')
            #print(objects)
            #print('^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^')
        if 'procedure' in map_object_types:
            logger.log('','Getting current Snowflake PROCEDURES to calculate diffs from config')
            for db in dbs:
                procedure_new = wrangle.wrangle_procedure(db['DATABASE_NAME'], config['ENV_PROCEDURE_PREFIX'], config['ENV_DATABASE_PREFIX'], config['ENV_ROLE_PREFIX'], config['DEPLOY_DATABASE_NAME'], ignore_roles_list, config['DEPLOY_TAGS'],config['DEPLOY_ROLE'], available_roles, config['HANDLE_OWNERSHIP'],semaphore)
                procedures = procedures + procedure_new
            db_hash_dict['procedure'] = hsh.hash_procedure_all(procedures)

        if 'function' in map_object_types:
            logger.log('','Getting current Snowflake FUNCTIONS to calculate diffs from config')
            for db in dbs:
                function_new = wrangle.wrangle_function(db['DATABASE_NAME'], config['ENV_FUNCTION_PREFIX'], config['ENV_DATABASE_PREFIX'], config['ENV_ROLE_PREFIX'], config['DEPLOY_DATABASE_NAME'], ignore_roles_list, config['DEPLOY_TAGS'],config['DEPLOY_ROLE'], available_roles, config['HANDLE_OWNERSHIP'],semaphore)
                functions = functions + function_new
            db_hash_dict['function'] = hsh.hash_function_all(functions)

        if 'task' in map_object_types:
            logger.log('','Getting current Snowflake TASKS to calculate diffs from config')
            for schema in schemas:
                task_new = wrangle.wrangle_task(schema['DATABASE_NAME'], schema['SCHEMA_NAME'], config['ENV_DATABASE_PREFIX'], config['ENV_ROLE_PREFIX'], config['DEPLOY_DATABASE_NAME'], ignore_roles_list, config['DEPLOY_TAGS'],config['DEPLOY_ROLE'], available_roles, config['HANDLE_OWNERSHIP'],semaphore)
                tasks = tasks + task_new
            db_hash_dict['task'] = hsh.hash_task_all(tasks)

        if 'masking_policy' in map_object_types:
            logger.log('','Getting current Snowflake MASKING POLICIES to calculate diffs from config')
            for schema in schemas:
                masking_policy_new = wrangle.wrangle_masking_policy(schema['DATABASE_NAME'], schema['SCHEMA_NAME'], config['ENV_DATABASE_PREFIX'], config['ENV_ROLE_PREFIX'], config['DEPLOY_DATABASE_NAME'], ignore_roles_list, config['DEPLOY_TAGS'],config['DEPLOY_ROLE'], available_roles, config['HANDLE_OWNERSHIP'],semaphore)
                masking_policies = masking_policies + masking_policy_new
            db_hash_dict['masking_policy'] = hsh.hash_masking_policy_all(masking_policies)

        if 'row_access_policy' in map_object_types:
            logger.log('','Getting current Snowflake ROW ACCESS POLICIES to calculate diffs from config')
            for schema in schemas:
                row_access_policy_new = wrangle.wrangle_row_access_policy(schema['DATABASE_NAME'], schema['SCHEMA_NAME'], config['ENV_DATABASE_PREFIX'], config['ENV_ROLE_PREFIX'], config['DEPLOY_DATABASE_NAME'], ignore_roles_list, config['DEPLOY_TAGS'],config['DEPLOY_ROLE'], available_roles, config['HANDLE_OWNERSHIP'],semaphore)
                row_access_policies = row_access_policies + row_access_policy_new
            db_hash_dict['row_access_policy'] = hsh.hash_row_access_policy_all(row_access_policies)

        #for obj in map_object_types:
        #    object_type = obj['object_type']
            #if object_type == 'database':
            #    logger.log('','Getting current Snowflake DATABASES to calculate diffs from config')
            #    dbs = wrangle.wrangle_database(config['ENV_DATABASE_PREFIX'], config['ENV_ROLE_PREFIX'], excluded_databases, config['DEPLOY_DATABASE_NAME'], ignore_roles_list, config['DEPLOY_TAGS'],config['DEPLOY_ROLE'], available_roles, config['HANDLE_OWNERSHIP'], import_databases,semaphore)
            #    db_hash_dict['database'] = hsh.hash_database_all(dbs)

            #elif object_type == 'schema':
            #    logger.log('','Getting current Snowflake SCHEMAS to calculate diffs from config')
            #    #print('------------------')
            #    #print(dbs)
            #    #print('------------------')
            #    for db in dbs:
            #        schemas_new = wrangle.wrangle_schema(db['DATABASE_NAME'], config['ENV_DATABASE_PREFIX'], config['ENV_ROLE_PREFIX'], config['DEPLOY_DATABASE_NAME'], ignore_roles_list, config['DEPLOY_TAGS'],config['DEPLOY_ROLE'], available_roles, config['HANDLE_OWNERSHIP'],semaphore)
            #        schemas = schemas + schemas_new
            #    db_hash_dict['schema'] = hsh.hash_schema_all(schemas)
            #    #print('+++++++++++++++++++')
            #    #print(db_hash_dict['schema'])
            #    #print('+++++++++++++++++++')
            #elif object_type == 'tag':
            #    logger.log('','Getting current Snowflake TAGS to calculate diffs from config')
            #    for schema in schemas:
            #        tags_new = wrangle.wrangle_tag(schema['DATABASE_NAME'], schema['SCHEMA_NAME'], config['ENV_DATABASE_PREFIX'], config['DEPLOY_DATABASE_NAME'], ignore_roles_list, config['DEPLOY_TAGS'],config['DEPLOY_ROLE'], available_roles, config['HANDLE_OWNERSHIP'],semaphore)
            #        tags = tags + tags_new
            #    db_hash_dict['tag'] = hsh.hash_tag_all(tags)
            #
            #elif object_type == 'object':

            #elif object_type == 'warehouse':
            #    logger.log('','Getting current Snowflake WAREHOUSES to calculate diffs from config')
            #    whs = wrangle.wrangle_warehouse(config['ENV_WAREHOUSE_PREFIX'], config['ENV_DATABASE_PREFIX'], config['ENV_ROLE_PREFIX'], config['DEPLOY_DATABASE_NAME'], ignore_roles_list, config['DEPLOY_TAGS'],config['DEPLOY_ROLE'], available_roles, config['HANDLE_OWNERSHIP'],semaphore)
            #    db_hash_dict['warehouse'] = hsh.hash_warehouse_all(whs)

            #elif object_type == 'role':
            #    logger.log('','Getting current Snowflake ROLES to calculate diffs from config')
            #    rs = wrangle.wrangle_role(config['ENV_ROLE_PREFIX'], config['ENV_DATABASE_PREFIX'], config['DEPLOY_DATABASE_NAME'], ignore_roles_list, config['DEPLOY_TAGS'],config['DEPLOY_ROLE'], available_roles, config['HANDLE_OWNERSHIP'],semaphore)
            #    db_hash_dict['role'] = hsh.hash_role_all(rs)

            #elif object_type == 'procedure':

            #elif object_type == 'function':

            #elif object_type == 'task':

            #elif object_type == 'masking_policy':

            #elif object_type == 'row_access_policy':

            #else:
            #    raise object_type_not_supported(object_type)

        completed = []  # need to track the objects already processed to know when downstream (next) processes can begin based on all dependencies being "complete"
        processing = []  # need to track currently executing processes for race conditions when a process kicks off (from a next of another object) that is already processing to avoid duplicate execution
        # Create a bunch of threads and add to array
        # NOTE: These threads have not actually began to execute yet
        threads = []
        #for i in range(10):
        for ref_id in map_start:
            tn = 'thread_' + ref_id
            t = threading.Thread(target=task, name=tn, args=(semaphore, tn, ref_id, map, completed, processing, deploy, base_rendered_path, config, logger, available_roles, val, object_state_dict, db_hash_dict))
            threads.append(t)

        # Kick off all threads to begin execution
        for t in threads:
            t.start()

        # if threads are not joined, the script will continue to run while the threads are also running.
        # Joining the threads means we will wait here until all the threads are done processing before continuing
        for t in threads:
            t.join()

        while len(threading.enumerate()) > 1:
            #print('Active processes: ' + str(len(threading.enumerate())))
            sleep(1)

        # Once all threads are complete

        logger.log('','all deployments have completed')
        msg = "--- Executed in %s seconds ---" % (round(time.time() - start_time,1))
        logger.log('',msg)

    except Exception as ex:
        #print(str(ex))
        traceback_text = traceback.format_exc() # get error traceback
        msg = 'error - see error log at end'
        logger.log('',msg)
        logger.log_error(str(ex), 'script', traceback_text)

    finally:
        if 'sf' in locals() or 'sf' in globals():
            del sf

        error_count = logger.get_error_count()
        if error_count > 0:
            logger.show_all_errors()
            raise Exception("Errors occured - see log for details")
        else:
            logger.log('','No errors! Nice job!')


#!/usr/bin/python3
"""
This Python script fetches the last 10 commits of a repository ("rails") by
user ("rails") in GitHub using the GitHub API.
"""
from requests import get
from sys import argv

if __name__ == "__main__":
    """
    using GitHub API to fetch commit history of a repository
    """

    url = 'https://api.github.com/repos/{}/{}/commits'.format(argv[2], argv[1])
    response = get(url)
    lenOfCommits = len(response.json())
    commits = response.json()
    if lenOfCommits >= 10:
        lenOfCommits = 10
    for i in range(0, lenOfCommits):
        print("{}: {}".format(commits[i].get('sha'), commits[i].get('commit').
                              get('author').get('name')))


class Solution(object):
    def sortColors(self, nums):
        ''' method 1 bubble sort

        time: O(n^2)
        space: O(1)
        '''
        for i in range(0, len(nums)):
            for j in range(0, i):
                if nums[i] < nums[j]:
                    nums[i], nums[j] = nums[j], nums[i]
        return nums

        '''
        method 2

        实质上是 Dutch National Flag 问题
        link: https://en.wikipedia.org/wiki/Dutch_national_flag_problem

        初始化两个指针 l, r 分别指向最左和最右
        用 cur 遍历：当 nums[cur] 对应的值为 0 时，将 cur 跟 l 交换，右移 cur 和 l
                    当 nums[cur] 对应的值为 2 时，将 cur 跟 r 交换，左移 r
                    当 nums[cur] 对应的值为 1 时，将 cur 右移即可
        '''
        l = cur = 0
        r = len(nums) - 1
        while cur <= r:
            if nums[cur] == 0:
                nums[cur], nums[l] = nums[l], nums[cur]
                cur += 1
                l += 1
            elif nums[cur] == 2:
                nums[cur], nums[r] = nums[r], nums[cur]
                r -= 1
            elif nums[cur] == 1:
                cur += 1

        '''
        method 3 internal function
        '''
        from collections import Counter
        return sorted(Counter(nums).elements())

s = Solution()
print(s.sortColors([2, 0, 2, 1, 1, 0, 0, 2, 2]))


#!/usr/bin/python3
"""
    Python file similar to model_state.py named model_city.py
    that contains the class definition of a City.
    Script 14-model_city_fetch_by_state.py that prints all City
    objects from the database hbtn_0e_14_usa:
"""

from sys import argv
from model_state import Base, State
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker

if __name__ == "__main__":
    """
        Accesses to the database and get the cities
        from the database
    """

    db_uri = 'mysql+mysqldb://{}:{}@localhost:3306/{}'.format(
            argv[1], argv[2], argv[3])

    engine = create_engine(db_uri)
    Session = sessionmaker(bind=engine)

    session = Session()
    outcome = session.query(City, State).join(State)

    for city, state in outcome.all():
        print("{}: ({}) {}".format(state.name, city.id, city.name))

    session.commit()
    session.close()


import torch
import torch.nn as nn
import torch.nn.functional as F
import os
import sys
import gc
sys.path.append("..")
from utils import data_loader,utils
import argparse
import random
import pickle as pkl
from tqdm import tqdm
import datetime
from time import time
from algo import GraphHINGE_FFT,GraphHINGE_Conv,GraphHINGE_Cross,GraphHINGE_CrossConv,GraphHINGE_ALL, GraphHINGE_ConvALL, GraphHINGE_CNN
from tensorboardX import SummaryWriter
import numpy as np
from sklearn.metrics import average_precision_score, roc_auc_score

class dummy_context_mgr():
    def __enter__(self):
        return None
    def __exit__(self, exc_type, exc_value, traceback):
        return False

def train(model, device, optimizer, train_loader, eval_loader, save_dir, epochs, writer, model_name, patience):
    best_val_acc = 0.0
    cnt = 0
    inference = eval_loader is None
    for epoch in range(epochs):
        #train
        model.train()
        train_loss = []
        pbar = tqdm(train_loader)
        train_acc = []
        for i, data in enumerate(pbar):
            UI, IU, UIUI, IUIU, UIAI1, IAIU1, UIAI2, IAIU2, UIAI3, IAIU3, labels = data
            optimizer.zero_grad()
            pred = model(UI.to(device), IU.to(device),\
            UIUI.to(device), IUIU.to(device), \
            UIAI1.to(device), IAIU1.to(device), \
            UIAI2.to(device), IAIU2.to(device), \
            UIAI3.to(device), IAIU3.to(device))

            loss = criterion(pred, labels.to(device))
            acc = utils.evaluate_acc(pred.detach().cpu().numpy(), labels.numpy())
            train_acc.append(acc)
            loss.backward()
            optimizer.step()

            train_loss.append(loss.item())
            pbar.set_description('Epoch {:d} | Loss {:.4f}'.format(epoch + 1, loss))
            train_iter= epoch * len(train_loader) + i
            if train_iter%10 ==0:
                writer.add_scalar('Loss/train', loss.item(), train_iter)

            '''
            if i%500==0:
                print('Epoch {:d} | Batch {:d} | Train Loss {:.4f} | '.format(epoch + 1, i+1,loss))

            del UI, IU, UIUI, IUIU, UIAI1, IAIU1, UIAI2, IAIU2, UIAI3, IAIU3, labels, pred
            gc.collect()
            with torch.cuda.device(device):
                torch.cuda.empty_cache()
            '''
        train_acc = np.mean(train_acc)
        writer.add_scalar('Acc/Train', train_acc, train_iter)
        #eval
        if inference:
            continue
        model.eval()
        eval_loss = []
        eval_acc = []
        eval_auc = []
        eval_logloss = []
        eval_f1 = []
        with torch.no_grad():
            pbar = tqdm(eval_loader)
            for i, data in enumerate(pbar):
                UI, IU, UIUI, IUIU, UIAI1, IAIU1, UIAI2, IAIU2, UIAI3, IAIU3, labels = data

                pred = model(UI.to(device), IU.to(device),\
                UIUI.to(device), IUIU.to(device), \
                UIAI1.to(device), IAIU1.to(device), \
                UIAI2.to(device), IAIU2.to(device), \
                UIAI3.to(device), IAIU3.to(device))

                loss = criterion(pred, labels.to(device))
                acc = utils.evaluate_acc(pred.detach().cpu().numpy(), labels.numpy())
                eval_loss.append(loss.item())
                eval_acc.append(acc)
                pbar.set_description('Epoch {:d} | Loss {:.4f}'.format(epoch + 1, loss))

                '''
                del UI,IU, UIUI, IUIU, UIAI1, IAIU1, UIAI2, IAIU2, UIAI3, IAIU3, labels, pred
                gc.collect()
                with torch.cuda.device(device):
                    torch.cuda.empty_cache()
                '''

            eval_loss = np.mean(eval_loss)
            eval_acc = np.mean(eval_acc)

            print('Epoch {:d} | Eval Acc {:.4f} | '.format(epoch + 1, eval_acc))
            writer.add_scalar('Loss/Eval', eval_loss, train_iter)
            writer.add_scalar('Acc/Eval', eval_acc, train_iter)
            if eval_acc > best_val_acc:
                print(f"Prev best Acc {best_val_acc:.4f}, new best {eval_acc:.4f}. Saving best weights...")
                best_val_acc = eval_acc
                best_epoch = epoch
                torch.save(model.state_dict(), os.path.join(save_dir,model_name))
                cnt = 0
            else:
                cnt = cnt + 1
                if cnt >= patience:
                    print("Early stopping")
                    print("best epoch:{}".format(best_epoch))
                    break
    if inference:
        torch.save(model.state_dict(), os.path.join(save_dir,model_name))

def test(model, device, test_loader, writer, inference_output = None):
    test_loss = []
    test_logloss = []
    pred_list = None
    label_list = None
    model.eval()
    with torch.no_grad():
        for i, data in enumerate(tqdm(test_loader, desc = 'Infer')):
            UI, IU, UIUI, IUIU, UIAI1, IAIU1, UIAI2, IAIU2, UIAI3, IAIU3, labels = data
            pred = model(UI.to(device), IU.to(device),\
            UIUI.to(device), IUIU.to(device), \
            UIAI1.to(device), IAIU1.to(device), \
            UIAI2.to(device), IAIU2.to(device), \
            UIAI3.to(device), IAIU3.to(device))

            loss = criterion(pred, labels.to(device))

            pred = pred.detach().cpu().numpy()
            labels = labels.numpy()
            users = UI.detach().cpu().numpy()[:, 0, 0]
            items = IU.detach().cpu().numpy()[:, 0, 0]
            if pred_list is None:
                pred_list = pred
                label_list = labels
            else:
                pred_list = np.vstack([pred_list, pred])
                label_list = np.vstack([label_list, labels])
            if inference_output is not None:
                for user, item, logit, label in zip(users, items, pred, labels):
                    print(user, item, logit[0].item(), sep = '\t', file = inference_output)
            logloss = utils.evaluate_logloss(pred, labels)
            test_loss.append(loss.item())
            test_logloss.append(logloss)
            writer.add_scalar('Loss/Test', loss.item(), i)
            writer.add_scalar('Logloss/Test', logloss, i)
            '''

            del UI, IU, UIUI, IUIU, UIAI1, IAIU1, UIAI2, IAIU2, UIAI3, IAIU3, labels, pred
            gc.collect()

            with torch.cuda.device(device):
                torch.cuda.empty_cache()
            '''
        test_auc = utils.evaluate_auc(pred_list, label_list)
        test_acc = utils.evaluate_acc(pred_list, label_list)
        test_aupr = average_precision_score(label_list, pred_list)

        test_f1 = utils.evaluate_f1_score(pred_list, label_list)

        test_loss = np.mean(test_loss)
        test_logloss = np.mean(test_logloss)
        writer.add_scalar('AUC/Test', test_auc, i)
        writer.add_scalar('Acc/Test', test_acc, i)
        writer.add_scalar('F1/Test', test_f1, i)
        print('Test Loss {:.4f} | Test AUC {:.4f} | Test ACC {:.4f} | Test F1 {:.4f} | AUPR {:.4f} | Test Logloss {:.4f} |'.format(test_loss, test_auc, test_acc, test_f1, test_aupr, test_logloss))

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Parameters.')
    parser.add_argument('-cuda', type=str, default='cpu' ,help='Cuda number.')
    parser.add_argument('-hidden', type=int, default=128, help='Hidden Units.')
    parser.add_argument('-heads', type=int, default=3, help='Attention heads.')
    parser.add_argument('-lr', type=float, default=0.001, help='Learning rate.')
    parser.add_argument('-wd', type=float, default=0.0009, help='Weight decay.')
    parser.add_argument('-epochs', type=int, default=500,help='Maximum Epoch.')

    parser.add_argument('-model', type=str, default='GraphHINGE_FFT',help='Model.')
    parser.add_argument('-save_dir', type=str, default='../out',help='Trained models to be saved.')
    parser.add_argument('-pretrained', type=str, default='pretrained.pth',help='Load a model from breakpoint.')

    parser.add_argument('-d', type=str,help='Dataset.')
    parser.add_argument('-p', type=str,help='Dataset path.')
    parser.add_argument('-o', type=str,default='../data/data_out',help='Output path.')
    parser.add_argument('-b', type=int, default=128, help='Batch size.')
    parser.add_argument('-s', type=int,default=0, help='Sample path to be saved.')
    parser.add_argument('-n', type=int, default=16,help='Num of walks per node.')
    parser.add_argument('-w', type=int, default=1,help='Scale of walk length.')
    parser.add_argument('-temp1', type=float, default=0.2, help='Temperature factor for node att.')
    parser.add_argument('-temp2', type=float, default=0.2, help='Temperature factor for path att.')
    parser.add_argument('-ratio', type=float, default=1, help='Sample ratio.')
    parser.add_argument('-train', action= "store_true", default= False,help='If false, model will be loaded from the latest saved one.')
    parser.add_argument('-usersid', type=int, action="append", default=None, help='Specify users id to inference. Default all users.')
    parser.add_argument('-itemsid', type=int, action="append", default=None, help='Specify items id to inference. Default all items.')
    parser.add_argument('-inference', action= "store_true", default=False,help='Inference stage. (Train All data and inference new links between users and items)')
    parser.add_argument('-model_state_file', type=str,default = None, help='Load model state file to infer')
    parser.add_argument('-use_tensorboard', action= "store_true",default = False)
    parser.add_argument('-kfold', action= "store_true",default = False)
    args = parser.parse_args()
    device = torch.device(args.cuda)

    fold_idx = -1 if args.kfold else None
    args.o += "_fold" if args.kfold else ""
    args.o += "_inference" if args.inference else ""

    for user_metas, item_metas, train_loader, eval_loader, test_loader, user_num, item_num, attr1_num, attr2_num, attr3_num in \
        data_loader.Dataloader(args.p, args.d, args.o, args.s, args.n, args.w, args.b, args.inference,ratio=args.ratio, usersid = args.usersid, itemsid = args.itemsid, kfold = args.kfold).load_data():
        print('Data Loaded!')
        if args.kfold:
            fold_idx += 1
        if args.model == 'GraphHINGE_FFT':
            model = GraphHINGE_FFT.GraphHINGE(
                user_num, item_num, attr1_num, attr2_num, attr3_num, args.hidden, args.hidden, args.hidden, args.heads, args.temp1, args.temp2, output_attentions = False
                ).to(device)
        elif args.model == 'GraphHINGE_Conv':
            model = GraphHINGE_Conv.GraphHINGE(
                user_num, item_num, attr1_num, attr2_num, attr3_num, args.hidden, args.hidden, args.hidden, args.heads, args.temp1, args.temp2
                ).to(device)
        elif args.model == 'GraphHINGE_Cross':
            model = GraphHINGE_Cross.GraphHINGE(
                user_num, item_num, attr1_num, attr2_num, attr3_num, args.hidden, args.hidden, args.hidden, args.heads, args.temp1, args.temp2
                ).to(device)
        elif args.model == 'GraphHINGE_CrossConv':
            model = GraphHINGE_CrossConv.GraphHINGE(
                user_num, item_num, attr1_num, attr2_num, attr3_num, args.hidden, args.hidden, args.hidden, args.heads, args.temp1, args.temp2
                ).to(device)
        elif args.model == 'GraphHINGE_ALL':
            model = GraphHINGE_ALL.GraphHINGE(
                user_num, item_num, attr1_num, attr2_num, attr3_num, args.hidden, args.hidden, args.hidden, args.heads, args.temp1, args.temp2
                ).to(device)
        elif args.model == 'GraphHINGE_ConvALL':
            model = GraphHINGE_ConvALL.GraphHINGE(
                user_num, item_num, attr1_num, attr2_num, attr3_num, args.hidden, args.hidden, args.hidden, args.heads, args.temp1, args.temp2
                ).to(device)
        elif args.model == 'GraphHINGE_CNN':
            model = GraphHINGE_CNN.GraphHINGE_CNN(
                user_num, item_num, attr1_num, attr2_num, attr3_num, args.hidden, args.hidden, args.hidden, args.heads, args.temp1, args.temp2
                ).to(device)
        else:
            raise NotImplementedError
        print('Model Created!')

        if (os.path.exists(os.path.join(args.save_dir, args.pretrained))):
            model.load_state_dict(torch.load(os.path.join(args.save_dir, args.pretrained)))
            print("Model loaded!")

        criterion = nn.BCELoss()
        optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.wd)
        if args.model_state_file is None:
            model_name = "NotFFT" + '_'.join([args.model,
                                        args.d, ("inference" if args.inference else "train"),
                                        f"Fold{fold_idx}" if args.kfold else "Nofold",
                                        "lr_"+str(args.lr), "bs_"+str(args.b), "wd_"+str(args.wd)]) \
                                    + '.pth'
        else:
            model_name = args.model_state_file

        save_dir = args.save_dir
        startDateTime = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
        log_dir = save_dir+'/Logs/'+ startDateTime + model_name
        print("Args ", args)
        if not os.path.exists(log_dir):
            os.mkdir(log_dir)
        with  SummaryWriter(log_dir = log_dir) as writer:
            if args.train:
                train(model, device, optimizer, train_loader, eval_loader, args.save_dir, args.epochs, writer, model_name,patience=50)
                pass
            model.load_state_dict(torch.load(os.path.join(args.save_dir, model_name)))
            if args.inference:
                if not os.path.exists(os.path.join(save_dir, "inference")):
                    os.mkdir(os.path.join(save_dir, "inference"))
                with open(os.path.join(save_dir, "inference", f"{startDateTime}.csv"), 'w') as infer_out:
                    test(model, device, test_loader, writer, infer_out)
            else:
                test(model, device, test_loader, writer)


from flask import Flask, render_template, request
from search1 import score, retrieve, loadandbuildindex2, createDataInvertIndex, build_index, build_indexMy
from time import time
import os

app = Flask(__name__, template_folder='.')

# -- get df -- #

"""
try:
    file_path = str()
    for roots, dirs, files in os.walk('./'):
        for file in files:
            if file.find('lyric') != -1:
                print('find!')
                file_path = roots + '/' + file
    print('file path is : {}'.format(file_path))
    # -- Why this cannot load -- #
    df = pd.read_csv(file_path)
except:
    print('Did not read dataframe!')
"""
loadandbuildindex2()

@app.route('/', methods=['GET'])
def index():
    start_time = time()
    query = request.args.get('query')
    if query is None:
        query = ''
    # return some documents
    documents = retrieve(query)
    # тут уже определяет некоторое ранжирование (сортируем)
    # doc - some feature (and returned all data)
    documents = sorted(documents, key=lambda doc: -score(query, doc))
    # для каждого документа меняем его формат добавляя строку с присвоенным значением величины ранга
    #results = [doc.format(query)+['%.2f' % score(query, doc)] for doc in documents]
    results = [doc.format(query)+['%.2f' % score(query, doc)] for doc in documents]

    return render_template(
        'index.html',
        time="%.2f" % (time()-start_time),
        query=query,
        search_engine_name='Yandex',
        results=results
    )

if __name__ == '__main__':
    app.run(debug=True, host='127.0.0.1', port=5004)


#!/usr/bin/env python
# coding: utf-8

# In[3]:

import spacy
from spacy.lang.en.stop_words import STOP_WORDS
from string import punctuation
from heapq import nlargest

# In[4]:

from newspaper import Article   #뉴스기사 text크롤링 해주는 라이브러리

# In[97]:

url = 'https://www.nytimes.com/2023/04/09/world/europe/europe-shells-ukraine-ammunition.html'

# In[98]:

article = Article(url)

# In[99]:

article.download()

# In[100]:

article.parse()

# In[101]:

newsTitle = article.title

# In[102]:

newstext= article.text

# In[115]:

# summarize code

def summarize(text,per):
    nlp = spacy.load('en_core_web_sm')
    doc = nlp(text)
    tokens = [token.text for token in doc]
    word_frequencies = {}
    for word in doc:
        if word.text.lower() not in list(STOP_WORDS):
            if word.text.lower() not in punctuation:
                if word.text not in word_frequencies.keys():
                    word_frequencies[word.text]=1
                else:
                    word_frequencies[word.text] += 1
    max_frequency = max(word_frequencies.values())
    for word in word_frequencies.keys():
        word_frequencies[word]= word_frequencies[word]/max_frequency
    sentence_tokens = [sent for sent in doc.sents]
    sentence_scores = {}
    for sent in sentence_tokens:
        for word in sent:
            if word.text.lower() in word_frequencies.keys():
                sentence_scores[sent]=word_frequencies[word.text.lower()]
            else:
                sentence_scores[sent]+=word_frequencies[word.text.lower()]

    select_length = int(len(sentence_tokens)*per)
    summary = nlargest(select_length,sentence_scores,key=sentence_scores.get)
    final_summary =[word.text for word in summary]
    summary = ''.join(final_summary)
    return summary

# In[116]:

# 번역 코드

import googletrans

translator = googletrans.Translator()

instr = '안녕하세요'

outstr = translator.translate(instr,dest='en', src = 'auto')

print(f"{instr} => {outstr.text}")

# In[117]:

summarized_text = summarize(newstext,0.05)
summarized_text

# In[ ]:


from django.conf.urls import url
from payment import views

urlpatterns = [
    url(r'^$', views.Payment_Manager.Launch, name='payment'),
    url(r'^potwierdzenie/$', views.Apply_Payment.Launch, name='payment.apply'),
    url(r'^anulowanie/$', views.Cancel_Payment.Launch, name='payment.cancel'),
    url(r'^dotpay/$', views.DotPay.Service, name='payment.dotpay'),
    url(r'^paypal/$', views.PayPal.Service, name='payment.paypal'),
    url(r'^kup_teraz/(?P<pk>\d+)/$', views.Buy.Buy_Product, name='payment.buy'),
]


import os
import discord
from discord.ext import commands
import json
import random
from discord.ext.commands import Bot

with open('C:\\Users\\alann\\Documents\\GitHub\dc\\setting.json','r',encoding='utf8') as jfile:
    jdata = json.load(jfile)

intents = discord.Intents.all()

bot = commands.Bot(command_prefix='?',intents=intents)

@bot.event
async def on_ready():
    print("on_ready")
    channel = bot.get_channel(int(jdata['LAMYBOT']))
    await channel.send("こんらみ!!!")

for filename in os.listdir('C:\\Users\\alann\\Documents\\GitHub\\dc\\cmd'):
    if filename.endswith('.py'):
        bot.load_extension(f'cmd.{filename[:-3]}')

if __name__ == "__main__":
    bot.run(jdata['TOKEN'])


#%%
import os
import numpy as np
import pandas as pd
import seaborn as sns
import plotly.express as px
from IPython.display import display
from matplotlib import pyplot as plt
from umap import UMAP
from sklearn.decomposition import PCA
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import roc_auc_score
from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_score
from maayanlab_bioinformatics.dge import characteristic_direction
from maayanlab_bioinformatics.normalization import log2_normalize, filter_by_var, zscore_normalize
from maayanlab_bioinformatics.utils import merge

#%%
df_data = pd.read_csv('data.csv', index_col=0)
df_data.columns = df_data.columns.astype(str)
df_data.columns.name = 'Barcode'
metadata = pd.read_csv('metadata.csv', index_col=0)
metadata.index = metadata.index.astype(str)
metadata['Labels'] = metadata['Labels'].astype(str)

#%%
df_library_size = pd.DataFrame(
    {
        'n_reads': df_data[df_data > 0].count(),
        'log_n_reads': np.log2(df_data[df_data > 0].count() + 1),
        'n_expressed_genes': df_data.sum(),
    }
).sort_values('n_reads', ascending=False)

display(df_library_size.head())
sns.distplot(df_data.iloc[0, :]); plt.show()
sns.distplot(df_data.iloc[:, 0]); plt.show()

#%%
df_data_norm = filter_by_var(df_data)
df_data_norm = log2_normalize(df_data_norm)
df_data_norm = zscore_normalize(df_data_norm)

#%%
sns.distplot(df_data_norm.iloc[0, :]); plt.show()
sns.distplot(df_data_norm.iloc[:, 0]); plt.show()

#%%
data_norm_pca = PCA(
  random_state=42,
)
data_norm_pca.fit(df_data_norm.values.T)
df_data_norm_pca = pd.DataFrame(
    data_norm_pca.transform(df_data_norm.values.T),
    index=df_data_norm.T.index
)
df_data_norm_pca.columns = [
    f'PCA-{c}' # ({r:.3f})'
    for c, r in zip(df_data_norm_pca.columns, data_norm_pca.explained_variance_ratio_)
]

#%%
px.scatter(
  merge(
    df_data_norm_pca,
    df_library_size,
    metadata,
  ),
  x=df_data_norm_pca.columns[0],
  y=df_data_norm_pca.columns[1],
  size='n_reads',
  size_max=8,
  color='Labels',
  hover_data=[df_data_norm.columns],
)

#%%
data_norm_umap = UMAP(
  random_state=42,
  n_components=2,
  n_neighbors=30,
  metric='cosine',
  min_dist=0.3,
)
data_norm_umap.fit(df_data_norm_pca.iloc[:, :10].values)
df_data_norm_umap = pd.DataFrame(
  data_norm_umap.transform(df_data_norm_pca.iloc[:, :10].values),
  columns=['UMAP-1', 'UMAP-2'],
  index=df_data_norm_pca.index,
)

#%%
px.scatter(
  merge(
    df_data_norm_umap,
    df_library_size,
    metadata,
  ),
  x=df_data_norm_umap.columns[0],
  y=df_data_norm_umap.columns[1],
  size='n_reads',
  size_max=8,
  color='Labels',
  hover_data=[df_data_norm.columns],
)

#%%
silhouette_scores = {}
for n in range(2, 25):
    y_pred = KMeans(n_clusters=n, random_state=42).fit_predict(df_data_norm_umap.values)
    silhouette_scores[n] = silhouette_score(df_data_norm_umap.values, y_pred, metric='cosine')

silhouette_scores = pd.DataFrame([
    {'N Clusters': k, 'Silhouette Score': v}
    for k, v in silhouette_scores.items()
])
best = silhouette_scores.sort_values('Silhouette Score').iloc[-1]
silhouette_scores

#%%
plt.plot(silhouette_scores['N Clusters'], silhouette_scores['Silhouette Score'])
plt.scatter([best['N Clusters']], [best['Silhouette Score']], label='Best')
plt.legend()
plt.title('Cluster size selection')
plt.ylabel('Silhouette Score')
plt.xlabel('Number of Clusters')
plt.show()

#%%
km = KMeans(n_clusters=int(best['N Clusters']), random_state=42)
df_data_norm_km = pd.DataFrame({
    'Cluster': [
        str(c)
        for c in km.fit_predict(df_data_norm_umap.values)
    ]
}, index=df_data_norm_umap.index)

#%%
# Perform differential expression for each cluter
diff_expr = {}
for cluster, samples in df_data_norm_km.groupby('Cluster'):
  diff_expr[cluster] = characteristic_direction(
    # expression outside of this cluster
    df_data_norm.loc[:, df_data_norm.columns.difference(samples.index)],
    # expression in this cluster
    df_data_norm.loc[:, samples.index],
  )['CD-coefficient']

df_diff_expr = pd.DataFrame(diff_expr)
df_diff_expr.index.name = 'Feature Name'

#%%
df_diff_expr['Cluster 0'].sort_values(ascending=True)
#%%
aucs = {}
for cluster, samples in df_data_norm_km.groupby('Cluster'):
  aucs[cluster] = {}
  for feature in metadata.columns:
    lr = LogisticRegression()
    X = metadata.loc[:, feature].values[:, np.newaxis]
    y_true = (df_data_norm_km['Cluster'] == cluster).astype(np.int64)
    lr.fit(X, y_true)
    y_score = lr.predict_proba(X)[:, 1]
    aucs[f"Cluster {cluster} CD"][feature] = roc_auc_score(y_true, y_score)

pd_aucs = pd.DataFrame(aucs)

#%%
# /clustering/graphclust/clusters.csv
#   Barcode,Cluster
os.makedirs('clustering/graphclust', exist_ok=True)
df_data_norm_km.to_csv('clustering/graphclust/clusters.csv')
# /diffexp/graphclust/differential_expression.csv
#   Feature Name,Cluster 2 Log2 fold change,Cluster 0 Log2 fold change,Cluster 1 Log2 fold change,Cluster 3 Log2 fold change,Cluster 2 Adjusted p value,Cluster 0 Adjusted p value,Cluster 1 Adjusted p value,Cluster 3 Adjusted p value,Cluster 2 Mean Counts,Cluster 0 Mean Counts,Cluster 1 Mean Counts,Cluster 3 Mean Counts,Feature Name
os.makedirs('diffexp/graphclust', exist_ok=True)
df_diff_expr.to_csv('diffexp/graphclust/differential_expression.csv')
# /pca/10_components/projection.csv
#   Barcode,PC-1,PC-2,PC-3,PC-4,PC-5,PC-6,PC-7,PC-8,PC-9,PC-10
os.makedirs('pca/10_components', exist_ok=True)
df_data_norm_pca.to_csv('pca/10_components/projection.csv')
# /umap/2_components/projection.csv
#   Barcode,UMAP-1,UMAP-2
os.makedirs('umap/2_components', exist_ok=True)
df_data_norm_umap.to_csv('umap/2_components/projection.csv')
# /cluster_aucs.csv
#   ,2,0,1,3
pd_aucs.to_csv('cluster_aucs.csv')


import numpy as np
import matplotlib.pyplot as plt
from matplotlib.widgets import Slider, Button, RadioButtons
from calc import Computer

fig, ax = plt.subplots()
plt.subplots_adjust(left=0.25, bottom=0.4)

labels = ['Prec Hate', 'Rec Hate', 'F1 Hate', 'Prec Off', 'Rec Off', 'F1 Off', 'F1 Macro']
x = np.arange(len(labels))

g0 = 0.16
hg0 = 0.8
hi0 = 0.4
ch0 = 0.5
co0 = 0.8

c = Computer(g0, hg0, hi0, ch0, co0)
values = c.values_from_labels(labels)
width = 0.2

bars = ax.bar(x, values, width, color='r', tick_label=labels)
ax.set_ylabel('Percentage')
ax.set_yticks(np.arange(0.0, 1.0, 0.1))

axcolor = 'lightgoldenrodyellow'
ax_g = plt.axes([0.25, 0.1, 0.65, 0.03], facecolor=axcolor)
ax_hg = plt.axes([0.25, 0.15, 0.65, 0.03], facecolor=axcolor)
ax_hi = plt.axes([0.25, 0.20, 0.65, 0.03], facecolor=axcolor)
ax_ch = plt.axes([0.25, 0.25, 0.65, 0.03], facecolor=axcolor)
ax_co = plt.axes([0.25, 0.30, 0.65, 0.03], facecolor=axcolor)

s_g = Slider(ax_g, 'P(Grp)', 0.0, 1.0, valinit=c._init_probs['g'])
s_hg = Slider(ax_hg, 'P(Hate|Grp)', 0.0, 1.0, valinit=c._init_probs['h_g'])
s_hi = Slider(ax_hi, 'P(Hate|Ind)', 0.0, 1.0, valinit=c._init_probs['h_i'])
s_ch = Slider(ax_ch, 'P(Correct|Hate)', 0.0, 1.0, valinit=c._init_probs['c_h'])
s_co = Slider(ax_co, 'P(Correct|Off)', 0.0, 1.0, valinit=c._init_probs['c_o'])

def update(val):
  c.update(s_g.val, s_hg.val, s_hi.val, s_ch.val, s_co.val)
  values = c.values_from_labels(labels)
  for rect, value in zip(bars, values):
    rect.set_height(value)
  fig.canvas.draw_idle()

s_g.on_changed(update)
s_hg.on_changed(update)
s_hi.on_changed(update)
s_ch.on_changed(update)
s_co.on_changed(update)

resetax = plt.axes([0.8, 0.025, 0.1, 0.04])
button = Button(resetax, 'Reset', color=axcolor, hovercolor='0.975')

def reset(event):
  s_g.reset()
  s_hg.reset()
  s_hi.reset()
  s_ch.reset()
  s_co.reset()

button.on_clicked(reset)

"""
rax = plt.axes([0.025, 0.5, 0.15, 0.15], facecolor=axcolor)
radio = RadioButtons(rax, ('red', 'blue', 'green'), active=0)

def colorfunc(label):
    l.set_color(label)
    fig.canvas.draw_idle()
radio.on_clicked(colorfunc)
"""

plt.show()


import sys
import numpy as np
import scipy.linalg as la
import scipy.sparse.linalg as las

if len(sys.argv)!=6:
    print("Use: cg.py <herm pos def matrix> <target> <x0> <#iterations> <tolerance>\n",file=sys.stderr)
    sys.exit(1)

A=np.array(sys.argv[1].split(',')).astype(int)
m=np.sqrt(A.shape[0]).astype(int)
A=A.reshape(-1,m)
k=np.array(sys.argv[2].split(',')).astype(int)
n=int(sys.argv[4])
x=np.zeros((m,n))
x[:,0]=np.array(sys.argv[3].split(',')).astype(int)
p=np.zeros((m,n))
r=np.zeros((m,n))
a=np.zeros(n)
b=np.zeros(n)
tol=float(sys.argv[5])

print("x[:,0]:",x[:,0])
p[:,0]=r[:,0]=k-A@x[:,0]
print("p[:,0]:",p[:,0])
print("r[:,0]:",r[:,0])
for i in np.arange(0,n-1):
    print("ronde %d",i)
    a[i]=(r[:,i] @ r[:,i])/( p[:,i] @ A @ p[:,i] )
    print("a[i]:",a[i])
    x[:,i+1]=x[:,i]+a[i]*p[:,i]
    print("x[:,i+1]:",x[:,i+1])
    sol=x[:,i+1]
    if la.norm(k-A@x[:,i+1])<tol:#*la.norm(k):
        print("break")
        break
    r[:,i+1]=r[:,i]-a[i]*A @ p[:,i]
    print("r[:,i+1]:",r[:,i+1])
    b[i]=( r[:,i+1] @ r[:,i+1] )/( r[:,i] @ r[:,i] )
    print("b[i]:",b[i])
    p[:,i+1]=r[:,i+1] + b[i]*p[:,i]
    print("p[:,i+1]:",p[:,i+1])
print("i:",i)
print("sol:",sol)

#A=np.array([2,-1,-1,2]).reshape(-1,2)
#lu,piv=la.lu_factor(A)
#x=la.lu_solve((lu,piv),grad)


import webapp2
import os
import jinja2
import json
import datetime
from math import sin, cos, sqrt, atan2, radians
from stop_models import Stop
from notification_models import Notification
#from seed_stops_db import seed_data
from twilio.rest import Client
from google.appengine.ext import ndb
import logging

my_file = open("app-secrets.json")
my_secret = my_file.read()
SECRETS_DICT = json.loads(my_secret)
my_file.close()

jinja_env = jinja2.Environment(
loader=jinja2.FileSystemLoader(os.path.dirname(__file__)),
extensions=['jinja2.ext.autoescape'],
autoescape=True)

stop_name = ""

#returns time in minutes (this is always an underestimate, moreso for longer routes)
def find_time_to_stop(lat1, lng1, lat2, lng2):
    # approximate radius of earth in km
    R = 6373.0
    bus_speed = 26.4

    lat1 = radians(abs(float(lat1)))
    lng1 = radians(abs(float(lng1)))
    lat2 = radians(abs(float(lat2)))
    lng2 = radians(abs(float(lng2)))

    delta_lng = lng2 - lng1
    delta_lat = lat2 - lat1

    a = sin(delta_lat / 2)**2 + cos(lat1) * cos(lat2) * sin(delta_lng / 2)**2
    c = 2 * atan2(sqrt(a), sqrt(1 - a))

    #distance in km
    distance = R * c

    #converts to miles
    distance = distance / 1.609344

    #converts distance to time required to get to the stop
    time_to_next_stop = distance * 60 / 24.6

    #multiplied by two since our model is ideal conditions
    return 2 * time_to_next_stop

def SendNotification(Notification, myStop):
    account_sid = SECRETS_DICT['twilio_account_sid']
    auth_token  = SECRETS_DICT['twilio_auth_token']

    client = Client(account_sid, auth_token)

    message = client.messages.create(
        to="+12136047704",
        from_="+14243583569",
        body="Your stop at %s is coming up!" % Notification.stop_name)

    print(message.sid)

class MainHandler(webapp2.RequestHandler):
    def get(self):
        main_page_template = jinja_env.get_template('templates/homepage.html')
        self.response.write(main_page_template.render(()))

class LoadDataHandler(webapp2.RequestHandler):
    def get(self):
        seed_data()

class CreateRouteHandler(webapp2.RequestHandler):
    def get(self):
        create_route_template = jinja_env.get_template('templates/create_route.html')
        logging.error("CreateRouteHandler")
        self.response.write(create_route_template.render(()))

class ViewRouteHandler(webapp2.RequestHandler):
    def get(self):
        view_route_template = jinja_env.get_template('templates/view_route.html')

        current_stop = self.request.get('current_stop')
        next_stop = self.request.get('next_stop')

        stop_name = current_stop

        current_stop_lat = Stop.query().filter(Stop.stop_name == current_stop).fetch()[0].stop_lat
        current_stop_lon = Stop.query().filter(Stop.stop_name == current_stop).fetch()[0].stop_lon

        next_stop_lat = Stop.query().filter(Stop.stop_name == next_stop).fetch()[0].stop_lat
        next_stop_lon = Stop.query().filter(Stop.stop_name == next_stop).fetch()[0].stop_lon

        time_to_next_stop = find_time_to_stop(current_stop_lat, current_stop_lon, next_stop_lat, next_stop_lon)

        time_to_next_stop = float(int(time_to_next_stop * 10))
        time_to_next_stop /=10

        notification = Notification(target_time = datetime.datetime.now() + datetime.timedelta(minutes=time_to_next_stop), stop_name = next_stop).put()

        template_vars = {
            'time_to_next_stop': time_to_next_stop,
        }
        self.response.write(view_route_template.render(template_vars))

class InformationHandler(webapp2.RequestHandler):
    def get(self):
        info_template = jinja_env.get_template('templates/information.html')
        self.response.write(info_template.render())

class NotificationHandler(webapp2.RequestHandler):
    def get(self):
        logging.info("Cron is working")
        two_minutes_ago = (datetime.datetime.now() + datetime.timedelta(minutes=2))
        notifications = Notification.query(Notification.target_time <= datetime.datetime.now()).fetch()
        for notification in notifications:
            logging.error("Notification sent")
            SendNotification(notification, stop_name)
            notifications.sent = True
            notifications.put()

class OnRouteHandler(webapp2.RequestHandler):
    def get(self):
        on_route_template = jinja_env.get_template('templates/on_route.html')
        self.response.write(on_route_template.render())

app = webapp2.WSGIApplication([
    ('/', MainHandler),
    ('/seed-data', LoadDataHandler),
    ('/create-route', CreateRouteHandler),
    ('/view-route', ViewRouteHandler),
    ('/info', InformationHandler),
    ('/notifier', NotificationHandler),
    ('/on-route', OnRouteHandler),
], debug=True)


import youtube_dl
import os

ydl_opts = {
    'format': 'bestaudio/best',
    'postprocessors': [{
        'key': 'FFmpegExtractAudio',
        'preferredcodec': 'mp3',
        'preferredquality': '192',
    }],
    'outtmpl': 'music/%(title)s.mp3',
    'quiet': False
}

def dl_file(url):
    with youtube_dl.YoutubeDL(ydl_opts) as ydl:
        ydl.download([url])
        result = ydl.extract_info(url, download=False)
        outfile = ydl.prepare_filename(result)
    return outfile


from datasets import kitti, mpo

def define_dataset(cfg, phase: str = "train", modality=["depth"]):
    if cfg.name == "kitti_odometry":
        dataset = kitti.KITTIOdometry(
            root=cfg.root,
            split=phase,
            shape=cfg.shape,
            min_depth=cfg.min_depth,
            max_depth=cfg.max_depth,
            flip=cfg.flip,
            modality=modality,
        )
    elif cfg.name == "mpo_sparse":
        dataset = mpo.MPOSparse(
            root=cfg.root,
            split=phase,
            shape=cfg.shape,
            min_depth=cfg.min_depth,
            max_depth=cfg.max_depth,
            flip=cfg.flip,
            modality=modality,
        )
    else:
        raise NotImplementedError
    return dataset


# https://leetcode.com/problems/house-robber/
# tags: #array, #blind_75_must_do, #dp, #top_interview_questions
from typing import List

class Solution:
    def rob(self, nums: List[int]) -> int:
        n = len(nums)
        max_amount = [0] * n
        max_amount[0] = nums[0]

        for i in range(1, n):
            prev = max_amount[i - 2] if i > 1 else 0
            max_amount[i] = max(max_amount[i - 1], prev + nums[i])

        return max_amount[-1]

if __name__ == "__main__":
    sol = Solution()
    print(sol.rob(nums=[1, 2, 3, 1]))  # 4


from django.core.management.base import BaseCommand, CommandError
from tooskie.populate import populate_db, populate_db_one_recipe

class Command(BaseCommand):
    help = 'Populate database from scrapped data'

    def add_arguments(self, parser):
        parser.add_argument(
            '--recipe',
            dest='recipe',
            help='Populate a specific recipe',
        )
    def handle(self, *args, **options):
        if 'recipe' in options and options['recipe']:
            for recipe_id in options['recipe']:
                populate_db_one_recipe(recipe_id)
        else:
            populate_db()


from django.shortcuts import redirect, render
from . models import JobModel
from django.core.paginator import Paginator
from . forms import ApplyForm, JobModelForm
from django.contrib.auth.decorators import login_required
from . filters import jobFilter

# Create your views here.

def home(request):
    context= {
        'jobs': JobModel.objects.all(),
        'jobsCount': JobModel.objects.count(),
    }
    return render(request, 'pages/home.html', context)

def jobList(request):

    jobsObject = JobModel.objects.all()
    myFilter = jobFilter(request.GET, queryset=jobsObject)
    jobsObject = myFilter.qs

    paginator = Paginator(jobsObject,5)
    pageNumber = request.GET.get('page')
    pageObject = paginator.get_page(pageNumber)

    context= {
        'jobs': pageObject,
        'jobsCount': JobModel.objects.count(),
        'myFilter' : myFilter
    }
    return render(request, 'pages/jobList.html', context)

def jobDetails(request, slug):
    jobObject = JobModel.objects.get(slug=slug)
    if request.method=="POST":
        form = ApplyForm(request.POST, request.FILES)
        if form.is_valid():
            myform = form.save(commit=False)
            myform.job = jobObject
            myform.save()

    else:
        form = ApplyForm

    context= {
        'job': jobObject,
        'form': form,
    }
    return render(request, 'pages/jobDetails.html', context)

@login_required
def postJob(request):

    if request.method == "POST":
        form = JobModelForm(request.POST, request.FILES )
        if form.is_valid():
            myform = form.save(commit=False)
            myform.user = request.user
            myform.save()
            return redirect("/")
    else:
        form = JobModelForm

    contex = {
        'form' : form
    }
    return render(request, 'pages/postJob.html', contex)


# -*- coding:utf-8 -*-
import random
import time

from utils.config import config
from utils.es_client import ElasticsearchClient
from utils.mq_client import RocketProducer, RocketPushConsumer

rocket_producer = RocketProducer({"host": "172.31.236.21", "port": 9876})
rocket_producer.start()
for i in range(10):
    data = {
        "id": random.randint(1, 10000),
        "trance_id": random.randint(1, 1000),
        "order_id": random.randint(1, 1000),
        "biz_job_id": "aaaa",
        "transfer_job_id": "bbbb",
        "move_job_id": "cccc",
        "agv_code": "dddd",
        "container_code": 9001001002,
        "log_level": "ERROR",
        "message": "abcdefg222222222222222",
        "created_time": int(round(time.time() * 1000)),
        "created_app": "wcs"
    }
    rocket_producer.push("log", data)
rocket_producer.shutdown()

# rocket_consumer = RocketPushConsumer({"host": "172.31.236.21", "port": 9876, "topic": "log"})
# es_instance = ElasticsearchClient(
#     {"host": config.HOST_IP1, "port": 9200, "index_name": config.ES_INDEX, "index_type": config.ES_TYPE})
# rocket_consumer.start(es_instance)
# from utils.config import config
# from utils.es_client import ElasticsearchClient
#
# es1 = ElasticsearchClient(
#     {"host": config.HOST_IP1, "port": 9200, "index_name": config.ES_INDEX, "index_type": config.ES_TYPE})
# print(es1)
# es2 = ElasticsearchClient(
#     {"host": config.HOST_IP1, "port": 9200, "index_name": config.ES_INDEX, "index_type": config.ES_TYPE})
# print(es2)
# print(id(es1))
# print(id(es2))


import scrapy
from pbl.items import PblSpider
from pbl.items import ShopCard
import json

class SpiderasortiSpider(scrapy.Spider):
    name = 'spiderAsorti'
    allowed_domains = ['www.assorti.lt']
    start_urls = ['https://www.assorti.lt/katalogas/maistas/']
    item = []
    list = [{
        'sid': 1,
        'name': 'Assorti',
        'domain': 'https://www.assorti.lt',
        'imageurl': 'https://www.assorti.lt/images/logo_white.png',
        'product': item
        }]

    def __init__(self):
        self.declare_xpath()

    def declare_xpath(self):
        self.getAllItemsXpath = '//*[@id="products_wrapper"]/div[2]/div/a/@href'
        self.TitleXpath  = '//*[@id="products_detailed"]/div[1]/div/div/div[2]/h1/text()'
        self.ImageXpath = '//*[@id="products_photos"]/div[1]/img/@src'
        self.PriceXpath = '//*[@id="products_add2cart"]/form/div/div[1]/div/div[1]/span/span[1]/text()'

    def parse(self, response):

        for href in response.xpath(self.getAllItemsXpath):
            url = response.urljoin(href.extract())
            yield scrapy.Request(url,callback=self.parse_item)

        next_page = response.xpath('//*[@id="products_wrapper"]/div[3]/div[2]/ul/li/a[contains(@class, "pagination_link")]/@href').extract()
        if next_page[1] is not '#':
            print('-' * 70)
            print(next_page[1])
            print('-' * 70)
            url = response.urljoin(next_page[1])
            yield scrapy.Request(url, callback=self.parse)

    def parse_item(self, response):

            shop = ShopCard()

            Title = response.xpath(self.TitleXpath).extract_first()
            Link = response.url
            Image = response.xpath(self.ImageXpath).extract_first()
            Price = response.xpath(self.PriceXpath).extract_first()
            Price = Price.replace(',', '.')
            Price = float(Price.split(' ')[0])

            shop['item'] = {
                'title': Title,
                'link': Link,
                'image': Image,
                'price': Price
            }

            self.item.append(shop['item'])

    def closed(self, reason):
        with open("spiderAsorti.json", "w") as final:
            json.dump(self.list, final, indent=2, ensure_ascii=False)


import falcon
from resources import athlete
from resources import plan
from resources import exercise
from services import database_service

from middlewares import (
    ContentEncodingMiddleware,
)

conn = database_service.connect()

api = falcon.API(middleware=[
    ContentEncodingMiddleware(),
])

# api = falcon.API()
athlete = athlete.Athlete(conn, database_service)
plan = plan.Plan(conn, database_service)
exercise = exercise.Exercise(conn, database_service)

api.add_route('/athletes/{id}', athlete)
api.add_route('/athletes', athlete, suffix='collection')
api.add_route('/plans/{id}', plan)
api.add_route('/plans', plan, suffix='collection')
api.add_route('/exercises/{id}', exercise)
api.add_route('/exercises', exercise, suffix='collection')


import binascii
from PIL import Image

new = Image.open('cat_with_secrets.png').load()
old = Image.open('cat_with_secrets_original.jpg').load()

width = 512

in_hex = ''

for y in range(3):
    for x in range(width):
        if (y, x) <= (2, 125) and new[x, y] != old[x, y]:
            in_hex += '%x' % (x % 16)

print(binascii.unhexlify(in_hex))


from __future__ import annotations

from typing import List

MAX_COPY = 0x7
MAX_SINGLE = 0x7

class RleNibbleCompressor:
    def __init__(self, uncompressed_data: bytes):
        if not isinstance(uncompressed_data, memoryview):
            uncompressed_data = memoryview(uncompressed_data)
        self.uncompressed_data = uncompressed_data

    def compress(self) -> bytes:
        """Compresses the input data"""
        self.compressed_data: List[int] = []
        buffer_single: List[int] = []
        i = 0
        while i < len(self.uncompressed_data):
            nb = self._search_max_seq(i)
            while nb > 3:
                self._write_singles(buffer_single)
                buffer_single = []
                copy = min(MAX_COPY, nb)
                self._write_copy(self.uncompressed_data[i], copy)
                i += copy
                nb -= copy
            if nb != 0:
                buffer_single.append(self.uncompressed_data[i])
                i += 1
        self._write_singles(buffer_single)
        compact_data = []
        for x in range(len(self.compressed_data) // 2):
            v = 0
            for i in range(2):
                v += self.compressed_data[x * 2 + i] * (16 ** (1 - i))
            compact_data.append(v)
        if len(self.compressed_data) % 2 != 0:
            compact_data.append(self.compressed_data[-1] * 16)
        return bytes(compact_data)

    def _search_max_seq(self, i):
        nibble = self.uncompressed_data[i]
        nb = 1
        while (
            i + nb < len(self.uncompressed_data)
            and self.uncompressed_data[i + nb] == nibble
        ):
            nb += 1
        return nb

    def _write_copy(self, nibble, nb_copy):
        self.compressed_data.append(nb_copy + 0x8)
        self.compressed_data.append(nibble)

    def _write_singles(self, buffer_single):
        while len(buffer_single) > 0:
            tmp = buffer_single[:MAX_SINGLE]
            self.compressed_data.append(len(tmp))
            self.compressed_data.extend(tmp)
            buffer_single = buffer_single[MAX_SINGLE:]


from bench.common import (
    SampleConfig,
    register_sample,
    SampleSource,
    SampleTag,
)
import numpy as np

def get_sample_config():
    return SampleConfig(
        args_cases=[(12, 12, 12), (16, 16, 16), (32, 32, 32)],
        requires_grad=[False] * 3,
        backward=[False] * 2,
        performance_iters=1000,
        save_timeline=False,
        source=SampleSource.ASSEMBLYAI,
        url="https://colab.research.google.com/drive/1IPpwx4rX32rqHKpLz7dc8sOKspUa-YKO#scrollTo=RVJs4Bk8FjjO",  # noqa
        tags=[
            SampleTag.ForLoop, SampleTag.Reduce, SampleTag.AdvancedIndexing,
            SampleTag.BuiltInDataStructure
        ],
    )

def gen_np_args(N, W, H):
    shape = (N, W, H)
    input1 = np.random.randint(0, 5, shape)
    input1 = input1.astype(np.float32)
    input2 = np.random.randint(0, 1, (N, N))
    input2 = input2.astype(np.float32)
    return [input1, input2]

register_sample(__name__, get_sample_config, gen_np_args)


from sys import stdin
from fractions import Fraction
input = lambda: stdin.readline().strip()

MOD = 1000000007

class Sphere:
    __slots__ = ['x', 'r']
    def __init__(self, x=0, r=0):
        self.x = x
        self.r = r
    def __lt__(self, other) -> bool:
        if self.x + self.r != other.x + other.r:
            return self.x + self.r < other.x + other.r
        return self.r > other.r

class Intersec:
    __slots__ = ['x', 'prev', 'next']
    def __init__(self, x=0, prev=-1, next=-1):
        self.x = x
        self.prev = prev
        self.next = next

n = int(input())
sph = [None for i in range(n)]
stk = [None for i in range(2*n)]
sz = 0

def is_intersec_in_sphere(p: Intersec, s: Sphere) -> bool:
    if p.prev != -1:
        x0 = sph[p.prev].x
        r0 = sph[p.prev].r
    else:
        x0 = sph[p.next].x
        r0 = sph[p.next].r

    ysq = r0**2 - (p.x-x0)**2
    return (s.x-p.x)**2 + ysq <= s.r**2

def sphere_intersec_x(a: Sphere, b: Sphere) -> Fraction:
    if a.x > b.x:
        a, b = b, a
    d = b.x - a.x
    return a.x + Fraction(a.r**2 + d**2 - b.r**2, 2*d)

def integrate_sphere(s: Sphere, a: Fraction, b: Fraction) -> Fraction:
    return -Fraction(1, 3)*(b**3-a**3) + s.x*(b**2-a**2) + (s.r**2-s.x**2)*(b-a)

def powmod(a: int, b: int, m: int) -> int:
    if b == 0:
        return 1
    half = powmod(a, b//2, m)
    if b%2 == 0:
        return half*half%m
    return half*half%m*a%m

for i in range(n):
    sph[i] = Sphere(*map(int, input().split()))

sph.sort()

stk[0] = Intersec(sph[0].x - sph[0].r, -1, 0)
stk[1] = Intersec(sph[0].x + sph[0].r, 0, -1)
sz = 2

for i in range(1, n):
    if sph[i-1].x + sph[i-1].r == sph[i].x + sph[i].r:
        continue
    if is_intersec_in_sphere(stk[0], sph[i]):
        stk[0] = Intersec(sph[i].x - sph[i].r, -1, i)
        stk[1] = Intersec(sph[i].x + sph[i].r, i, -1)
        sz = 2
        continue
    if not is_intersec_in_sphere(stk[sz-1], sph[i]):
        stk[sz] = Intersec(sph[i].x - sph[i].r, -1, i)
        stk[sz+1] = Intersec(sph[i].x + sph[i].r, i, -1)
        sz += 2
        continue

    low = 0
    high = sz-1
    while high - low > 1:
        mid = (low + high) // 2
        if is_intersec_in_sphere(stk[mid], sph[i]):
            high = mid
        else:
            low = mid

    sz = high
    if stk[sz-1].next != -1:
        stk[sz] = Intersec(sphere_intersec_x(sph[stk[sz-1].next], sph[i]),
                           stk[sz-1].next, i)
    else:
        stk[sz] = Intersec(sph[i].x - sph[i].r, -1, i)
    stk[sz+1] = Intersec(sph[i].x + sph[i].r, i, -1)
    sz += 2

res: Fraction = 0
for i in range(sz-1):
    if stk[i].next == -1:
        continue
    res += integrate_sphere(sph[stk[i].next], stk[i].x, stk[i+1].x)
print(res.numerator*powmod(res.denominator, MOD-2, MOD)%MOD)


import json
import os
import argparse

def main(output_folder: str, source: str, output_file: str):
    """
    Retrieve missing twitter handles.

    Args:
        output_folder (str): Path to folder with files named after the retrieved users.
        source (str): Path to file containing list of users to be retrieved.
        output_file (str): Path to output file where remaining users should be stored.
    """
    get_twitter_handle = lambda x: x.replace(".json", "")
    scraped_twitter_handles = list(map(get_twitter_handle, os.listdir(output_folder)))

    with open(source, "r") as f:
        source_twitter_handles = json.load(f)

    remaining_twitter_handles = set(source_twitter_handles) - set(
        scraped_twitter_handles
    )
    with open(output_file, "w") as f:
        json.dump(list(remaining_twitter_handles), f)

    print(
        f"Saved {len(remaining_twitter_handles)} remaining twitter handles to {output_file}."
    )

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Load user(s) followers.")
    parser.add_argument(
        "--output-folder",
        type=str,
        help="Path to folder containing files with naming like TWITTER_HANDLE.json.",
        required=True,
    )
    parser.add_argument(
        "--source",
        type=str,
        help="Path to file containing all original twitter handles to scrape for.",
        required=True,
    )
    parser.add_argument(
        "--output-file",
        type=str,
        help="Path to file where remaining twitter handles should be stored.",
        required=True,
    )

    args = parser.parse_args()
    output_folder = args.output_folder
    source = args.source
    output_file = args.output_file

    main(output_folder, source, output_file)


import numpy as np
import matplotlib.pyplot as plt
import scipy.linalg as sci

ti=0
tf=10.
x,h=np.linspace(0,1,100,retstep=True)
dt=0.000052
t=ti
i=0
us= np.where(x<0.5,x,1-x)

m=(dt/h**2)
line,=plt.plot(x,us)
A = np.array(np.zeros((98,98), dtype = np.float))
B = np.array(np.zeros((98,98), dtype = np.float))
for i in range(98):
    for j in range(98):
        if i==j:
            A[i][j]=(1+m)
            B[i][j]=1-m
        elif abs(i-j)==1:
            A[i][j]=-0.5*m
            B[i][j]=0.5*m

while t<tf:
    us[1:-1]=sci.solve(A,np.dot(B,us[1:-1]))
    t+=dt
    line.set_ydata(us)
    plt.pause(0.0001)

plt.show()
"""while t<tf:
    uis[1:-1]=us[1:-1]+m*(us[:-2]+us[2:]-2*us[1:-1])
    us[1:-1]=0.5*(m/(1-m))*(us[2:]-2*us[1:-1]+us[:-2]+uis[2:]+uis[:-2])
    t+=dt
    line.set_ydata(us)
    plt.pause(0.0001)

plt.show()
"""


from django.urls import path
from . import views
urlpatterns=[
    path('',views.home,name="home"),
    path(r'transfer/(?P<value>\d+)/$',views.modelview,name='modelview'),
    path('download/',views.download,name='download'),
    path('home/',views.returnhome,name='returnhome'),
    path('about/',views.about,name='about'),
    path('feedback/',views.feedback,name='feedback'),
    path('submit/',views.submit,name='submit'),
]