import pandas as pd
import numpy as np
import xmltodict

#https://betterprogramming.pub/analyze-your-icloud-health-data-with-pandas-dd5e963e902f

input_path = './apple_health_export/export.xml'
with open(input_path, 'r') as xml_file:
    input_data = xmltodict.parse(xml_file.read())

records_list = input_data['HealthData']['Record']

raw_df = pd.DataFrame(records_list)

step_dist_df = raw_df[raw_df['@type'].isin(['HKQuantityTypeIdentifierStepCount', 'HKQuantityTypeIdentifierDistanceWalkingRunning'])].copy()

step_dist_df['value'] = step_dist_df['@value'].astype(float)
step_dist_df['localdate'] = pd.to_datetime(step_dist_df['@startDate'].str[:10])

#adding anomoly correction

step_dist_df['value'] = np.where((step_dist_df['@type'] == 'HKQuantityTypeIdentifierDistanceWalkingRunning') & (step_dist_df['value'] > 2.0), 2.0, step_dist_df['value'])
step_dist_df['value'] = np.where((step_dist_df['@type'] == 'HKQuantityTypeIdentifierStepCount') & (step_dist_df['value'] > 5000.0), 5000.0, step_dist_df['value'])

step_df = step_dist_df[step_dist_df['@type'] == 'HKQuantityTypeIdentifierStepCount'].groupby('localdate', as_index=False)['value'].sum().rename(columns={'localdate':'Date', 'value': 'Steps'})

#Fill in missing dates
step_df.set_index("Date", inplace=True)
step_df = step_df.asfreq("D", fill_value=0).reset_index()

milage_df = step_dist_df[step_dist_df['@type'] == 'HKQuantityTypeIdentifierDistanceWalkingRunning'].groupby('localdate', as_index=False)['value'].sum().rename(columns={'localdate':'Date', 'value': 'Miles'})

#Fill in missing days
milage_df.set_index("Date", inplace=True)
milage_df = milage_df.asfreq("D", fill_value=0).reset_index()

#Can change this to be made from a dataframe later
# event_df = pd.DataFrame([
# 	{'Date': pd.to_datetime('2019-07-20'), 'Event': 'Augustin in NYC. 100 degree weather'},
# 	{'Date': pd.to_datetime('2019-07-21'),'Event': 'Augustin in NYC. 100 degree weather'},
# 	{'Date': pd.to_datetime('2017-07-01'),'Event': 'London with Titouan'},
# 	{'Date': pd.to_datetime('2017-07-02'),'Event': 'London with Titouan'},
# 	{'Date': pd.to_datetime('2017-07-03'),'Event': 'Wimbledon'}

# ])
# event_df = pd.read_csv('events.csv')
# event_df['Date'] = pd.to_datetime(event_df.Date)

# step_df = step_df.merge(event_df, how='left', on='Date')
# milage_df = milage_df.merge(event_df, how='left', on='Date')

step_df.to_csv('steps.csv', index=False)
milage_df.to_csv('milage.csv', index=False)


import warnings

import joblib
from django.shortcuts import render

from .forms import TextForm

warnings.filterwarnings('ignore')
import re

import joblib
import scipy
import torch
from torch.utils.data import Dataset

TOKEN_RE = re.compile(r'[\w\d]+')

def tokenize_text_simple_regex(txt, min_token_size=4):
    txt = txt.lower()
    all_tokens = TOKEN_RE.findall(txt)
    return [token for token in all_tokens if len(token) >= min_token_size]

def character_tokenize(txt):
    return list(txt)

def tokenize_corpus(texts, tokenizer=tokenize_text_simple_regex, **tokenizer_kwargs):
    return [tokenizer(text, **tokenizer_kwargs) for text in texts]

def vectorize_texts(tokenized_texts, word2id, word2freq, mode='tfidf', scale=True):
    assert mode in {'tfidf', 'idf', 'tf', 'bin'}

    result = scipy.sparse.dok_matrix((len(tokenized_texts), len(word2id)), dtype='float32')
    for text_i, text in enumerate(tokenized_texts):
        for token in text:
            if token in word2id:
                result[text_i, word2id[token]] += 1

    if mode == 'bin':
        result = (result > 0).astype('float32')

    elif mode == 'tf':
        result = result.tocsr()
        result = result.multiply(1 / result.sum(1))

    elif mode == 'idf':
        result = (result > 0).astype('float32').multiply(1 / word2freq)

    elif mode == 'tfidf':
        result = result.tocsr()
        result = result.multiply(1 / result.sum(1))
        result = result.multiply(1 / word2freq)

    if scale:
        result = result.tocsc()
        result -= result.min()
        result /= (result.max() + 1e-6)

    return result.tocsr()

class SparseFeaturesDataset(Dataset):
    def __init__(self, features):
        self.features = features

    def call(self):
        return torch.from_numpy(self.features.toarray()).float()

VECTORIZATION_MODE = 'tfidf'
model = joblib.load("model.pkl")
dataset = joblib.load("test_dataset.pkl")
vocabulary = joblib.load("vocabulary.pkl")
word_doc_freq = joblib.load("word_doc_freq.pkl")
new_model = joblib.load("model_new.pkl")

def basic(request):
    form = TextForm(request.GET)
    text = request.GET.get('text')
    if text:
        text = [tokenize_text_simple_regex(text)]
        text = vectorize_texts(text, vocabulary, word_doc_freq, mode=VECTORIZATION_MODE)
        object = SparseFeaturesDataset(text).call()
        prediction = int(model(object).argmax())
        from_0_to_10 = int(new_model(object).argmax())
    else:
        prediction = 'пока нет предсказаний'
        from_0_to_10 = 'пока нет предсказаний'
    context = {
        'form': form,
        'prediction': prediction,
        'from_0_to_10': from_0_to_10,
    }
    return render(request, 'form.html', context)


# Complete program File
# The base is from abstract_to_gene.py at D:/Desktop/scrapy/Genes/abstract_to_gene.py
# Gene name is extracted from D:/Desktop/scrapy/Genes/filterDB.csv, not gen_ali.txt
# Disease filenames are extracted from : 민규 파일이름
import datetime
import os
import matplotlib.pyplot as plt

category = ["allrDB", "nervDB", "respDB"] # Database categories used in the research
genes = []

f2 = open("D:/Desktop/scrapy/Genes/final_genelist.txt", "r") # fixed: genes without common words ex: not, nan
x = f2.readlines()
for ele in x:
    genes.append(ele.strip())

# genes = f2.read().split(",") # fixed

par = [] # Gene list with parenthesis
norm = [] # Gene list without parenthesis
del_par = [] # The list that has elements of list "par" without parenthesis

for i in genes:
    if "nan" not in i and "(" in i:
        par.append(i)
# list of genes with parenthesis: par

for j in genes:
    if "nan" not in j and "(" not in j:
        norm.append(j)
# list of genes without parenthesis : norm

for k in par:
    del_par.append(k.replace("(", " ").replace(")", " ").strip())
# print(del_par)
# Delete parenthesis of genes in par : del_par

begin_time = datetime.datetime.now()
print("Program starts at: ", datetime.datetime.now().strftime("%X")) # Time that the program is started
for database in category:
    f0 = open("D:/Desktop/scrapy/Pubmed/{}_shredded_list.txt".format(database), "r") # 질병 목록파일
    z = f0.readlines() #한줄씩 읽기

    for disease in z:
        gene_appeared = [] # temporary List of genes that appear in every disease. Has duplicates so will be made to dictinary
        freq_gene_appeared = {} # Dictionary of genes and the number of appearance.
        a = disease.replace("  \n", "").strip() # Disease name from file z has space and \n in it. We delete it with strip()
        f = open("D:/Desktop/scrapy/PubMed/{}_final/{}.txt".format(database, a), "r") # Open files with the format 'database' and 'a'
        y = f.readlines()
        print("File '{}' is read".format(a), "Time:", datetime.datetime.now().strftime("%X")) # Shows the time when a new file is read

        print("Time lapse: ", datetime.datetime.now()-begin_time)
        f2 = open("D:/Desktop/scrapy/PubMed/{}_output/{}_output.txt".format(database,a), "w")
        f2.close()
        print("File '{}_output' is made.".format(a))
        # Find genes in abstract from norm:
        for index, line in enumerate(y):
            if index % 4 == 1:
                v = line.replace("(", " ").replace(")", " ") # Exchange parenthesis to space of abstract and store at temporary variable v
                templist = [] # Make a temporary list for each iterable
                    # print("Line {} is read".format(index))
                for words in v.split(): # Match each words of abstract with norm and append it to templist
                    if words in norm:
                        templist.append(words)
                        gene_appeared.append(words)
                        continue # Move to next so it doesn't count multiple encounters in paper.
                    elif words.strip() in del_par: # Match each words of abstract with del_par  and append it to templist
                        templist.append(words)
                        gene_appeared.append(words)
                        continue
                    else:
                        pass
                if len(templist) != 0: # Return result when list is not empty
                    write_f2 = open("D:/Desktop/scrapy/PubMed/allrDB_output/{}_output.txt".format(a), "a")
                    write_f2.write(y[index - 1])
                    write_f2.write(y[index])
                    write_f2.write(y[index + 1])
                    write_f2.write("Genes found: "+ str(set(templist)) + "\n\n")
                    write_f2.close()
                    print("Paper title: ", y[index - 1])
                    print("Abstract: ", y[index])
                    print(y[index + 1])
                    print("Genes found: ", str(set(templist))) # Make templist to set to delete duplicates
                    print("")

        for ele in gene_appeared:
            if ele in freq_gene_appeared:
                freq_gene_appeared[ele] += 1
            else:
                freq_gene_appeared[ele] = 1
        plt.figure(figsize = (15, 6), dpi = 250) # Adjust size and resolution of plot
        plt.bar(range(len(freq_gene_appeared)), freq_gene_appeared.values(), align='center')

        if len(gene_appeared) < 15:
            plt.xticks(range(len(freq_gene_appeared)), list(freq_gene_appeared.keys()))
        else:
            plt.xticks(range(len(freq_gene_appeared)), list(freq_gene_appeared.keys()), fontsize = 3, rotation = 90) # Fontsize default is 12, rotation default is 0

        plt.title('{}'.format(a))
        plt.xlabel('Gene Name')
        plt.ylabel('Frequency')
        plt.savefig('D:/Desktop/scrapy/PubMed/allrDB_output/{}_output.png'.format(a))
        # plt.show()

        f3 = open("D:/Desktop/scrapy/PubMed/allrDB_output/{}_output.txt".format(a), "a")
        f3.write("Number of genes found: " + str(len(set(gene_appeared))) + "\n")
        f3.write("Genes appeared in disease '{}' : ".format(a) + str(set(gene_appeared)) + "\n\n")
        f3.write("Frequency of gene appearance: " + str(freq_gene_appeared))
        f3.close()


from django.core.mail import send_mail, EmailMultiAlternatives
from django.template.loader import get_template

def send_email_attachment(title, description, start_time, end_time, send_to, message):
    message = EmailMultiAlternatives(subject=title, body=description,from_email=request.user.email, to=[send_to])
    html_templ = get_template("cal/email.html").render({'title':title, 'description': description, "start_time":start_time, "end_time":end_time, "send_to":send_to, 'message':message,})
    message.attach_alternative(html_templ, "text/html")
    message.send()
    print("email sent")


import json

from typing import Dict, Union

from data.models import Character, Weapon

def get_character_stats(
        character: Character
) -> Dict[str, Dict[str, str]]:
    """
    Unloads character data from JSON and returns
    it as a dictionary of stats.

    :param Character character: Character model object.
    :return Dict stats: Data interpreted as dictionary.
    """

    stats = {
        'name': character.name,
        'element': character.element,
        'type': character.weapon,
        'hp': json.loads(character.hp),
        'atk': json.loads(character.atk),
        'def': json.loads(character.defense),
        'normal_attack': json.loads(character.normal_atk),
        'skill': json.loads(character.skill),
        'burst': json.loads(character.burst),
    }

    return stats

def get_weapon_stats(
        weapon: Weapon
) -> Dict[str, Dict[str, str]]:
    """
    Unloads weapon data from JSON and returns
    it as a dictionary of stats.

    :param Weapon weapon: Weapon model object.
    :return Dict stats: Data interpreted as dictionary.
    """

    stats = {
        'name': weapon.name,
        'type': weapon.type,
        'atk': json.loads(weapon.atk),
        weapon.substat_name: json.loads(weapon.substat_lv)
    }

    return stats

def calculate_skill_damage(
        character: Character,
        weapon: Weapon,
        data: Dict[str, Dict[str, str]],
        skill: Dict[str, str],
        level: str
) -> Dict[str, Dict[str, Union[int, float]]]:
    """
    Main calculation function. Processes one skill at a time.

    :param character: Current character.
    :param weapon: Current weapon.
    :param data: Dictionary of current character data.
    :param skill: Current skill data.
    :param level: Skill level
    :return Dict[str, Dict[str, Union[int, float]]]: Dictionary of
    calculated value.
    """

    skill_data = {}

    for move, stats in skill.items():
        stat = stats['Lv' + level]
        # Current implementation of damage skill recognition.
        if '%' in stat:
            split_stat = stat.split(' ')
            print(f"{move} {split_stat}")
            if 'HP' in split_stat:
                damage_stat = data['hp']
            elif 'DEF' in split_stat:
                damage_stat = data['def']
            else:
                damage_stat = data['atk']
            for dmg in split_stat:
                if '%' in dmg:
                    new_dmg = dmg.replace('%', ' ').split(' ')
                    damage = round(float(new_dmg[0]) * damage_stat / 100, 2)
                    if not skill_data.get(move):
                        skill_data[move] = str(damage)
                    else:
                        skill_data[move] += f" + {str(damage)}"
                    skill_data[move] += f"{new_dmg[1]}"
        else:
            if not skill_data.get(move):
                skill_data[move] = stats['Lv' + level]
            else:
                continue

    return skill_data

def get_attributes(
        character: Character,
        weapon: Weapon,
        query_params,
) -> Union[str, Dict]:
    """
    Sums initial character and weapon attributes.

    :param Character character: Character object.
    :param Weapon weapon: Weapon object.
    :param Dict query_params: Dictionary of query parameters.

    :return Dict response_data: Initial dictionary of response data.
    """

    if weapon.type != character.weapon:
        return "Selected character cannot wield this weapon!"

    response_data = {}

    character_level = query_params['character_level']
    character_data = get_character_stats(character)
    weapon_level = query_params['weapon_level']
    weapon_data = get_weapon_stats(weapon)

    response_data['character_name'] = character_data['name']
    response_data['element'] = character_data['element']
    response_data['weapon_type'] = character_data['type']
    response_data['weapon'] = weapon_data['name']
    response_data['hp'] = int(character_data['hp'][character_level])
    response_data['atk'] = round(
        float(character_data['atk'][character_level]) +
        float(weapon_data['atk'][weapon_level]), 2)
    response_data['def'] = round(
        float(character_data['def'][character_level]), 2)
    skills = {
        'normal_attack': character_data['normal_attack'],
        'skill': character_data['skill'],
        'burst': character_data['burst']
    }
    levels = {
        'normal_attack': query_params['attack_level'],
        'skill': query_params['skill_level'],
        'burst': query_params['burst_level']
    }
    response_data['skills'] = []

    for skill, level in levels.items():
        response_data['skills'].append(calculate_skill_damage(
            character, weapon, response_data, skills[skill], level
        ))

    return response_data


import os
import json
from flask import Flask
from flask import Blueprint
from visualizer.visualizer import FeatureExtractor
from visualizer import auth
from visualizer.visualizer_blueprint import visualizer_blueprint

def read_plugin_config(vis_config_file=None):
    """ Read the pulgin configuration JSON file from a path, if its None, uses a default configuration """
    if vis_config_file != None:
        file_path = vis_config_file
    else:
        file_path = os.path.dirname(os.path.abspath(__file__)) + "//visualizer.json"
    with open(file_path) as f:
        data = json.load(f)
    return data

def create_app(test_config=None):
    """Create and configure an instance of the Flask application."""
    app = Flask(__name__, instance_relative_config=True, static_folder='base/static')
    BASE_DIR = os.path.dirname(os.path.abspath(__file__))
    # read plugin configuration JSON file
    p_config = read_plugin_config()
    # initialize FeatureExtractor
    fe = FeatureExtractor(p_config)
    # set flask app parameters
    app.config.from_mapping(
        # a default secret that should be overridden by instance config
        SECRET_KEY="dev",
        # store the database in the instance folder
        DATABASE=os.path.join(BASE_DIR, "test.sqlite"),
        # plugin configuration from visualizer.json
        P_CONFIG = p_config,
        # visualizer instance with plugins already loaded
        FE = fe
    )
    if test_config is None:
        # load the instance config, if it exists, when not testing
        app.config.from_pyfile("config.py", silent=True)
    else:
        # load the test config if passed in
        app.config.update(test_config)

    # ensure the instance folder exists
    try:
        os.makedirs(app.instance_path)
    except OSError:
        pass

    #@app.route("/hello")
    #def hello():
    #    return "Hello, World!"

    # register the database commands
    from visualizer import db

    db.init_app(app)

    # get the output plugin template folder
    plugin_folder = fe.ep_output.template_path(p_config)
    # construct the blueprint with configurable plugin_folder
    vis_bp = visualizer_blueprint(plugin_folder)
    # register the blueprints
    app.register_blueprint(auth.bp)
    app.register_blueprint(vis_bp)

    # add the / route pointing to the index endpoint
    app.add_url_rule("/", endpoint="index")

    return app


import copy
import logging
from os import environ

from thrift.protocol import TBinaryProtocol
from thrift.transport import TSocket, TTransport

from .genthrift.hive_metastore import ThriftHiveMetastore
from .genthrift.hive_metastore.ttypes import Database, Table, FieldSchema, Partition, \
    DropPartitionsRequest, RequestPartsSpec

from .genthrift.hive_metastore.ttypes import NoSuchObjectException

SIMPLE_SERDE = 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
INPUT_FORMAT = 'org.apache.hadoop.mapred.TextInputFormat'
OUTPUT_FORMAT = 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
DEFAULT_PORT = 9083

class HMSClient(ThriftHiveMetastore.Client):
    __client = None
    __isOpened = False

    def __init__(self, iprot=None, oprot=None, host=None, port=None):
        self.logger = logging.getLogger(__name__)

        if not iprot:
            if not host:
                host = environ.get("HMS_HOST")

            if not host:
                host = 'localhost'

            if ':' in host:
                parts = host.split(':')
                host = parts[0]
                port = int(parts[1])

            if not port:
                port = environ.get("HMS_PORT")

            if not port:
                port = DEFAULT_PORT

            transport = TTransport.TBufferedTransport(TSocket.TSocket(host, int(port)))
            protocol = TBinaryProtocol.TBinaryProtocol(transport)
            super(HMSClient, self).__init__(protocol, oprot)
        else:
            super(HMSClient, self).__init__(iprot, oprot)

    def open(self):
        self._oprot.trans.open()
        self.__isOpened = True
        return self

    def __enter__(self):
        self.open()
        return self

    def close(self):
        self._oprot.trans.close()

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.close()

    @staticmethod
    def make_schema(params):
        """
        Produce field schema from list of parameters

        :param params: list of parameters or tuples
        :type params: list[str]
        :return: resulting field schema
        :rtype: list[FieldSchema]
        """
        schema = []
        for param in params:
            param_type = 'string'
            if ':' in param:
                parts = param.split(':')
                param_name = parts[0]
                param_type = parts[1] if parts[1] else 'string'
            else:
                param_name = param

            schema.append(FieldSchema(name=param_name, type=param_type, comment=''))

        return schema

    @staticmethod
    def parse_schema(schemas):
        """
        Convert list of FieldSchema objects in a list of name:typ strings

        :param schemas:
        :type schemas: list[FieldSchema]
        :return:
        """
        return map(lambda s: '{}\t{}'.format(s.name, s.type), schemas)

    @staticmethod
    def make_partition(table, values):
        """

        :param table:
        :type table: Table
        :param values:
        :type values: list[str]
        :return:
        :rtype: Partition
        """
        partition_names = [k.name for k in table.partitionKeys]
        if len(partition_names) != len(values):
            raise ValueError('Partition values do not match table schema')
        kv = [partition_name + '=' + value for partition_name, value
              in zip(partition_names, values)]

        sd = copy.deepcopy(table.sd)
        sd.location = sd.location + '/' + '/'.join(kv)

        return Partition(values=values, dbName=table.dbName, tableName=table.tableName, sd=sd)

    def add_partition(self, table, values):
        """
        Add partition

        :param table:
        :type table: Table
        :param values:
        :type values: list[str]
        """
        super(HMSClient, self).add_partition(self.make_partition(table, values))

    def check_for_named_partition(self, db_name, table_name, partition):
        try:
            self.get_partition_by_name(db_name, table_name, partition)
            return True
        except NoSuchObjectException:
            return False

    def drop_partitions(self, db_name, table_name, names, need_result=None):
        """
        Drop specified partitions from the table

        :param db_name: Database name
        :type db_name: str
        :param table_name:
        :type table_name: str
        :param names: Partition names
        :type names: list[str]
        :param need_result: If true, return drop results
        :return: drop results
        """
        if not names:
            return None
        return self.drop_partitions_req(DropPartitionsRequest(db_name, table_name,
                                                                       RequestPartsSpec(names), need_result))

    def drop_all_partitions(self, db_name, table_name, need_result=None):
        return self.drop_partitions(db_name, table_name,
                                    self.get_partition_names(db_name, table_name),
                                    need_result)

    def get_current_notification_id(self):
        return self.get_current_notificationEventId().eventId


import requests
# import exp_tab

def region_ua(reg_ua):

    https = "https://www.ukrposhta.ua/address-classifier-ws/"
    https_get = "get_regions_by_region_ua"
    headers = {'Accept': 'application/json', 'Authorization': 'Bearer 371e26f3-7f69-3972-9b3d-9236d45ad98b'}
    params_vi = (
        ('region_name_ru', reg_ua),
        # ('region_name_en', 'khm'),
    )
    resp = requests.get(https + https_get, headers=headers, params=params_vi)
    # print(resp.status_code)
    # print(resp.json())
    zm = resp.json()
    # print(resp.url)
    for a,b in zm.items():
        for c,d in b.items():
            for i in d:
                for e,f in i.items():
                    if e == 'REGION_ID':
                        print('REGION_ID', f)
                        region_id = f
    return (region_id)

def region_id_code(reg_id,reg_district_ua):
    https = "https://www.ukrposhta.ua/address-classifier-ws/"
    https_get = "get_districts_by_region_id_and_district_ua"
    headers = {'Accept': 'application/json', 'Authorization': 'Bearer 371e26f3-7f69-3972-9b3d-9236d45ad98b'}
    params_vi = (
        ('region_id', reg_id),
        ('district_ua', reg_district_ua),
    )
    resp = requests.get(https + https_get, headers=headers, params=params_vi)
    # print(resp.status_code)
    # print(resp.json())
    zm = resp.json()
    # print(resp.url)
    for a,b in zm.items():
        for c,d in b.items():
            for i in d:
                for e,f in i.items():
                    if e == 'DISTRICT_ID':
                        print('DISTRICT_ID', f)
                        district_id = f
    return (district_id)

#
def city_ua(reg_id_cod,reg_city_ua):
    https = "https://www.ukrposhta.ua/address-classifier-ws/"
    https_get = "get_city_by_region_id_and_district_id_and_city_ua"
    headers = {'Accept': 'application/json', 'Authorization': 'Bearer 371e26f3-7f69-3972-9b3d-9236d45ad98b'}
    params_vi = (
        ('district_id', reg_id_cod),
        ('city_ua', reg_city_ua),
    )
    resp = requests.get(https + https_get, headers=headers, params=params_vi)
    # print(resp.status_code)
    # print(resp.json())
    zm = resp.json()
    # print(resp.url)
    for a,b in zm.items():
        for c,d in b.items():
            for i in d:
                for e,f in i.items():
                    if e == 'CITY_ID':
                        print('CITY_ID', f)
                        city_id = f
    return (city_id)

def address(region_id,city_id,district_id,street_ua,shortstreettype_ua):
    https = "https://www.ukrposhta.ua/address-classifier-ws/"
    https_get = "get_street_by_region_id_and_district_id_and_city_id_and_street_ua"
    headers = {'Accept': 'application/json', 'Authorization': 'Bearer 371e26f3-7f69-3972-9b3d-9236d45ad98b'}
    params_vi = (
        ('district_id', district_id),
        ('region_id', region_id),
        ('city_id', str(city_id)),
        ('street_ua', street_ua),
    )
    resp = requests.get(https + https_get, headers=headers, params=params_vi)
    # print(resp.status_code)
    # print(resp.json())
    zm = resp.json()
    # print(resp.url)
    # print(zm)
    for a,b in zm.items():
        for c,d in b.items():
            for i in d:
                for e,f in i.items():
                    if e == 'SHORTSTREETTYPE_UA' and f == shortstreettype_ua :
                        for g, h in i.items():
                            if g == 'STREET_ID':
                                print('STREET_ID', h)
                                street_id = h
    return (street_id)

def postcode(street_id, housenumber):
    https = "https://www.ukrposhta.ua/address-classifier-ws/"
    https_get = "get_addr_house_by_street_id"
    headers = {'Accept': 'application/json', 'Authorization': 'Bearer 371e26f3-7f69-3972-9b3d-9236d45ad98b'}
    params_vi = (
        ('street_id', str(street_id)),
        ('housenumber', housenumber)
    )
    resp = requests.get(https + https_get, headers=headers, params=params_vi)
    # print(resp.status_code)
    # print(resp.json())
    zm = resp.json()
    # print(resp.url)
    # print(zm)
    for a,b in zm.items():
        for c,d in b.items():
            for i in d:
                for e,f in i.items():
                    if e == 'POSTCODE':
                        print('POSTCODE', f)
                        post_code = f
    return (post_code)

def main():
# 'DISTRICT_UA': "Кам'янець-Подільський", 'DISTRICT_ID': '894',
# 'DISTRICT_UA': 'Хмельницький', 'DISTRICT_ID': '1009',
# 'DISTRICT_UA': 'Шепетівський', 'DISTRICT_ID': '1010',
    reg_ua = 'хм'
    reg_district_ua = "Хмельницький"
    # reg_district_ua = "Кам'янець-Подільський"
    # reg_district_ua = "Шепетівський"
    reg_city_ua = 'хмель'
    street_ua = 'іпод'
    # shortstreettype_ua = 'вул.'
    shortstreettype_ua = 'пров.'
    housenumber = 2

    region_id=region_ua(reg_ua)
    # print(region_id)

    district_id=region_id_code(region_id,reg_district_ua)
    # print(district_id)

    city_id = city_ua(district_id,reg_city_ua)
    # print(city_id)

    street_id = address(region_id, city_id, district_id, street_ua, shortstreettype_ua)
    # print(street_id)

    post_code = postcode(street_id, housenumber)
    # print(post_code)

if __name__ == "__main__":
    main()


from typing import Optional

import einops
import equinox as eqx
from fieldx._src.domain.domain import Domain
import jax
import jax.numpy as jnp
from jaxtyping import Array

from somax._src.models.qg.domain import LayerDomain
from somax._src.models.qg.params import QGParams
from somax._src.operators.dst import (
    inverse_elliptic_dst,
    laplacian_dst,
)

class DSTSolution(eqx.Module):
    homsol: Array = eqx.static_field()
    homsol_mean: Array = eqx.static_field()
    H_mat: Array = eqx.static_field()
    capacitance_matrix: Optional[Array] = eqx.static_field()

def calculate_helmholtz_dst(
    domain: Domain, layer_domain: LayerDomain, params: QGParams
) -> Array:
    # get Laplacian dst transform
    # print(domain.Nx, domain.dx)
    L_mat = laplacian_dst(
        domain.Nx[0] - 2, domain.Nx[1] - 2, domain.dx[0], domain.dx[1]
    )
    # print_debug_quantity(L_mat, "L_MAT")
    # get beta term
    lambda_sq = einops.rearrange(layer_domain.lambda_sq, "Nz -> Nz 1 1")
    beta = params.f0**2 * lambda_sq

    # calculate helmholtz dst
    H_mat = L_mat - beta

    return H_mat

def compute_homogeneous_solution(u: Array, lambda_sq: Array, H_mat: Array):
    # create constant field
    constant_field = jnp.ones_like(u)

    # get homogeneous solution
    sol = jax.vmap(inverse_elliptic_dst, in_axes=(0, 0))(
        constant_field[..., 1:-1, 1:-1], H_mat
    )

    # calculate the homogeneous solution
    homsol = constant_field + sol * lambda_sq

    return homsol


########################################################################################################################
########################################################################################################################
#                                              SEGMENTATION DES POUMONS                                                #
########################################################################################################################
########################################################################################################################

import os
import numpy as np
import pandas as pd
from random import sample
from keras.utils import to_categorical
from keras.callbacks import EarlyStopping
import model
import data
import utils
import sys

# Choix de path
console = False
if console:
    ROOT_DIR = os.path.abspath("/home/achauviere/Bureau/Projet_Detection_Metastase_Souris/deepmeta-master/")
    sys.path.append(ROOT_DIR)
    PATH_GIT = os.path.join(ROOT_DIR, "../deepmeta-master/")
    PATH_DATA = os.path.join(ROOT_DIR, "../DATA/")
    PATH_Synth = os.path.join(ROOT_DIR, "../Data_Synthetique/")
else:
    PATH_GIT = "../deepmeta-master/"
    PATH_DATA = "../DATA/"
    PATH_Synth = "../Data_Synthetique/"

path_souris = os.path.join(PATH_DATA, "Poumons/Souris/")
path_mask = os.path.join(PATH_DATA, "Poumons/Masques/")
path_img = os.path.join(PATH_DATA, "Poumons/Image/")
path_lab = os.path.join(PATH_DATA, "Poumons/Label/")
tab = pd.read_csv(os.path.join(PATH_DATA, "Poumons/Tableau_General.csv")).values

# Ensemble de Souris présente pour l'entrainement :
numSouris = utils.calcul_numSouris(path_souris)

####################################################################################
  ##################### MODELE 2D DETECTION + SEGMENTATION #####################
####################################################################################

"""
Detection Poumons :
"""

data_seg, label_seg, ind_seg = data.create_data_detect_poum(path_img, tab, numSouris)

no = np.arange(len(ind_seg)) ; no_sample = sample(list(no), len(no))

data_seg = data_seg.reshape(-1, 128, 128, 1)[no_sample].astype('float32')
label_seg = to_categorical(label_seg[no_sample])

model_detect = model.model_detection()
es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=3)
model_detect.fit(data_seg, label_seg, validation_split=0.2, batch_size=264, epochs=20, callbacks=[es])
model_detect.save(os.path.join(PATH_GIT, "Poumons/model/model_detect.h5"))

"""
Segmentation Poumons :
"""

data_2D, label_2D, ind_2D = data.create_data_seg_poum(path_img, path_lab, tab)

N = np.arange(len(ind_2D)) ; N_sample = sample(list(N), len(N))

data_2D = data_2D.reshape(-1, 128, 128, 1)[N_sample]
label_2D = label_2D.reshape(-1, 128, 128,1)[N_sample]

input_shape = (128,128,1)

model_seg = model.model_unet_2D(input_shape)
earlystopper = EarlyStopping(patience=5, verbose=1)
model_seg.fit(data_2D, label_2D, validation_split=0.2, batch_size=32, epochs=50,callbacks=[earlystopper])
model_seg.save(os.path.join(PATH_GIT, "Poumons/model/model_seg.h5"))

####################################################################################
        ##################### MODELE 2D MULTI-AXES #####################
####################################################################################

"""
Multi-Axes
    Chargement des images de souris annotées + amélioration du contraste
    Chargement des masques + complétions par masque vide
"""

data_3D, label_3D, ind_3D = data.crate_data_3D(path_img, path_lab, tab, numSouris)

#Data selon axe
data_ax = []; data_sag = []; data_cor = []
label_ax = []; label_sag = []; label_cor = []

for k in np.arange(len(numSouris)):

    for i in np.arange(128):
        data_ax.append(data_3D[k][i, :, :])
        label_ax.append(label_3D[k][i, :, :])

        data_sag.append(data_3D[k][:, i, :])
        label_sag.append(label_3D[k][:, i, :])

        data_cor.append(data_3D[k][:, :, i])
        label_cor.append(label_3D[k][:, :, i])

n = np.arange(len(ind_3D)) ; n_sample = sample(list(n), len(n))

data_ax = np.array(data_ax).reshape(-1, 128, 128, 1)[n_sample]
label_ax = np.array(label_ax, dtype=np.bool).reshape(-1, 128, 128, 1)[n_sample]

data_sag = np.array(data_sag).reshape(-1, 128, 128, 1)[n_sample]
label_sag = np.array(label_sag, dtype=np.bool).reshape(-1, 128, 128, 1)[n_sample]

data_cor = np.array(data_cor).reshape(-1, 128, 128, 1)[n_sample]
label_cor = np.array(label_cor, dtype=np.bool).reshape(-1, 128, 128, 1)[n_sample]

input_shape = (128, 128, 1)

## Modele axial
earlystopper = EarlyStopping(patience=5, verbose=1)
model_axial = model.model_unet_2D(input_shape)
model_axial.fit(data_ax, label_ax, validation_split=0.2, batch_size=32, epochs=50,
                                callbacks=[earlystopper])
model_axial.save(os.path.join(PATH_GIT, "Poumons/model/model_axial.h5"))

## Modele sagital
earlystopper = EarlyStopping(patience=5, verbose=1)
model_sagital = model.model_unet_2D(input_shape)
model_sagital.fit(data_sag, label_sag, validation_split=0.2, batch_size=32, epochs=50,
                  callbacks=[earlystopper])
model_sagital.save(os.path.join(PATH_GIT, "Poumons/model/model_sagital.h5"))

## Modele corronal
earlystopper = EarlyStopping(patience=5, verbose=1)
model_corronal = model.model_unet_2D(input_shape)
model_corronal.fit(data_cor, label_cor, validation_split=0.2, batch_size=32, epochs=50,
                     callbacks=[earlystopper])
model_corronal.save(os.path.join(PATH_GIT, "Poumons/model/model_corronal.h5"))

####################################################################################
        ##################### MODELE RESNET #####################
####################################################################################

resnet = model.ResNet50(input_shape=(128, 128, 1))
resnet.compile(optimizer='adam', loss='binary_crossentropy', metrics=[utils.mean_iou])
# En faite, il est possible de construire un U-Net avec encodeur resNet avec la librairie segmentation_models
# Cela peut être utile à essayer.

####################################################################################
        ##################### UNET PLUS PLUS #####################
####################################################################################

data_2D, label_2D, ind_2D = data.create_data_seg_poum(path_img, path_lab, tab)

N = np.arange(len(ind_2D)) ; N_sample = sample(list(N), len(N))

data_2D = data_2D.reshape(-1, 128, 128, 1)[N_sample]
label_2D = label_2D.reshape(-1, 128, 128, 1)[N_sample]

input_shape = (128, 128, 1)

model_seg = model.unet_plusplus(input_shape)
earlystopper = EarlyStopping(patience=5, verbose=1)
model_seg.fit(data_2D, label_2D, validation_split=0.2, batch_size=32, epochs=50,callbacks=[earlystopper])
model_seg.save(os.path.join(PATH_GIT, "model/model_unet_plusplus.h5"))

####################################################################################
        ##################### Small U-Net #####################
####################################################################################

data_2D, label_2D, ind_2D = data.create_data_seg_poum(path_img, path_lab, tab)

N = np.arange(len(ind_2D)) ; N_sample = sample(list(N), len(N))

data_2D = data_2D.reshape(-1, 128, 128, 1)[N_sample]
label_2D = label_2D.reshape(-1, 128, 128, 1)[N_sample]

input_shape = (128,128,1)

# model_seg = model.small_unet(input_shape)
model_seg = model.unetCoupe2Max(input_shape)

earlystopper = EarlyStopping(patience=5, verbose=1)
model_seg.fit(data_2D, label_2D, validation_split=0.2, batch_size=32, epochs=50,callbacks=[earlystopper])

# model_seg.save(os.path.join(PATH_GIT, "Poumons/model/small_unet.h5"))
model_seg.save(os.path.join(PATH_GIT, "Poumons/model/unetCoupe2Max.h5"))

####################################################################################
    ##################### AMELIORATION AVEC CREATIVE DATA #####################
####################################################################################

"""
Segmentation Poumons :
"""

# Charge les anciennes images
data_2D, label_2D, ind_2D = data.create_data_seg_poum(path_img, path_lab, tab)

# Charge les nouvelles images
path_new = os.path.join(PATH_Synth, "Nouvelles_Images/")
newData, newPoum, newMeta = data.recup_new_data(path_new)

# On concatene le tout
Data = utils.concat_data(data_2D, newData)
MaskPoum = utils.concat_data(label_2D, newPoum)

# Puis on procède comme précedemment
N = np.arange(Data.shape[0]) ; N_sample = sample(list(N), len(N))

Data = Data.reshape(-1, 128, 128, 1)[N_sample]
MaskPoum = MaskPoum.reshape(-1, 128, 128, 1)[N_sample]

input_shape = (128, 128, 1)

model_seg = model.unet_plusplus(input_shape)
earlystopper = EarlyStopping(patience=5, verbose=1)
model_seg.fit(Data, MaskPoum, validation_split=0.2, batch_size=32, epochs=50,callbacks=[earlystopper])
model_seg.save(os.path.join(PATH_GIT, "Poumons/model/model_seg_poum_creat.h5"))


import logging
import requests
import datetime
from config import weather_token
from aiogram import Bot, Dispatcher, executor, types
import config
from sqllighter import SQLighter

logging.basicConfig(level=logging.INFO)
bot = Bot(token=config.bot_token, parse_mode="HTML")
dp = Dispatcher(bot)
db = SQLighter('db.db')

@dp.message_handler(commands=['subscribe'])
async def subscribe(message: types.Message):
    if not db.subscriber_exists(message.from_user.id):
        db.add_subscriber(message.from_user.id)
    else:
        db.update_subscription(message.from_user.id, True)
    await message.answer("Спасибо за подписку на наш Telegramm канал! С нами будет весело!")

@dp.message_handler(commands=['unsubscribe'])
async def subscribe(message: types.Message):
    if not db.subscriber_exists(message.from_user.id):
        db.add_subscriber(message.from_user.id, False)
        await message.answer(
            "Очень жаль, что вы отписались, надеемся, что вы сообщите создателю о вашей проблеме: https://vk.com/augusstt")
    else:
        ()

@dp.message_handler(commands=['start'])
async def subscribe(message: types.Message):
    await message.reply("Привет, что ты именно хочешь?")

@dp.message_handler(commands=['weather'])
async def start_command(message: types.Message):
    await message.reply("Напиши мне название города, в котором хочешь узнать погоду")

@dp.message_handler()
async def start_command(message: types.Message):
    code_to_smile = {"Clear": "Ясно \U00002600",
                     "Clouds": "Облачно \U00002601",
                     "Rain": "Дождь \U00002614",
                     "Drizzle": "Дождь \U00002614",
                     "Thunderstorm": "Гроза \U000026A1",
                     "Snow": "Снег \U0001F328",
                     "Mist": "Туман \U0001F32B"}
    try:
        r = requests.get(
            f"http://api.openweathermap.org/data/2.5/weather?q={message.text}&appid={weather_token}&units=metric")
        data = r.json()
        city = data["name"]
        cur_weather = data["main"]["temp"]
        wind = data["wind"]["speed"]
        sunrise_timestamp = datetime.datetime.fromtimestamp(data["sys"]["sunrise"])
        sunset_timestamp = datetime.datetime.fromtimestamp(data["sys"]["sunset"])
        length_of_the_day = datetime.datetime.fromtimestamp(data["sys"]["sunrise"]) - datetime.datetime.fromtimestamp(
            data["sys"]["sunset"])
        weather_description = data["weather"][0]["main"]
        if weather_description in code_to_smile:
            wd = code_to_smile[weather_description]
        await message.reply(f"***{datetime.datetime.now().strftime('%Y-%m-%d %H:%M')}***\n"
                            f"Погода в городе: {city}\n Температура: {cur_weather}C° {wd}\n"
                            f"Ветер: {wind}\n"
                            f"Время рассвета: {sunrise_timestamp}\nВремя заката: {sunset_timestamp}\n"
                            f"Продолжительность дня: {length_of_the_day}")

    except:
        await message.reply("\U00002620 Проверьте название города \U00002620")
        return 0

if __name__ == '__main__':
    executor.start_polling(dp, skip_updates=True)


import torch.utils.data as data
from PIL import Image
import os
import os.path
import random
import errno
import codecs
import numpy as np
import torch
import torchvision.transforms as transforms
from utils import noisify

# Dataset for Fashion-MNIST
class F_MNIST(data.Dataset):
    urls = [
        'http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-images-idx3-ubyte.gz',
        'http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-labels-idx1-ubyte.gz',
        'http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-images-idx3-ubyte.gz',
        'http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-labels-idx1-ubyte.gz',
    ]
    raw_folder = 'f_raw'
    processed_folder = 'f_processed'
    training_file = 'training.pt'
    test_file = 'test.pt'

    def __init__(self, root, train=True, transform=None, target_transform=None, download=False, noise_type=None,
                 noise_rate=0.2, random_state=100):
        self.root = os.path.expanduser(root)
        self.transform = transform
        self.target_transform = target_transform
        self.train = train  # training set or test set
        self.dataset = 'f_mnist'
        self.noise_type = noise_type

        if download:
            self.download()

        if not self._check_exists():
            raise RuntimeError('Dataset not found.' +
                               ' You can use download=True to download it')

        if self.train:
            self.train_data, self.train_labels = torch.load(
                os.path.join(self.root, self.processed_folder, self.training_file))

            if noise_type != 'clean':
                self.train_labels = np.asarray([[self.train_labels[i]] for i in range(len(self.train_labels))])
                self.train_noisy_labels, self.actual_noise_rate = noisify(train_labels=self.train_labels,
                                                                          noise_type=noise_type, noise_rate=noise_rate,
                                                                          random_state=random_state)
                self.train_noisy_labels = [i[0] for i in self.train_noisy_labels]
                _train_labels = [i[0] for i in self.train_labels]
                self.noise_or_not = np.transpose(self.train_noisy_labels) == np.transpose(_train_labels)
        else:
            self.test_data, self.test_labels = torch.load(
                os.path.join(self.root, self.processed_folder, self.test_file))

    def __getitem__(self, index):

        if self.train:
            # if self.noise_type is not None:
            if self.noise_type != 'clean':
                img, target = self.train_data[index], self.train_noisy_labels[index]
            else:
                img, target = self.train_data[index], self.train_labels[index]
        else:
            img, target = self.test_data[index], self.test_labels[index]

        img = Image.fromarray(img.numpy(), mode='L')

        if self.transform is not None:
            img = self.transform(img)

        if self.target_transform is not None:
            target = self.target_transform(target)

        return img, target, index

    def __len__(self):
        if self.train:
            return len(self.train_data)
        else:
            return len(self.test_data)

    def _check_exists(self):
        return os.path.exists(os.path.join(self.root, self.processed_folder, self.training_file)) and \
               os.path.exists(os.path.join(self.root, self.processed_folder, self.test_file))

    def download(self):
        from six.moves import urllib
        import gzip

        if self._check_exists():
            return

        # download files
        try:
            os.makedirs(os.path.join(self.root, self.raw_folder))
            os.makedirs(os.path.join(self.root, self.processed_folder))
        except OSError as e:
            if e.errno == errno.EEXIST:
                pass
            else:
                raise

        for url in self.urls:
            print('Downloading ' + url)
            data = urllib.request.urlopen(url)
            filename = url.rpartition('/')[2]
            file_path = os.path.join(self.root, self.raw_folder, filename)
            with open(file_path, 'wb') as f:
                f.write(data.read())
            with open(file_path.replace('.gz', ''), 'wb') as out_f, \
                    gzip.GzipFile(file_path) as zip_f:
                out_f.write(zip_f.read())
            os.unlink(file_path)

        # process and save as torch files
        print('Processing...')

        training_set = (
            read_image_file(os.path.join(self.root, self.raw_folder, 'train-images-idx3-ubyte')),
            read_label_file(os.path.join(self.root, self.raw_folder, 'train-labels-idx1-ubyte'))
        )
        test_set = (
            read_image_file(os.path.join(self.root, self.raw_folder, 't10k-images-idx3-ubyte')),
            read_label_file(os.path.join(self.root, self.raw_folder, 't10k-labels-idx1-ubyte'))
        )
        with open(os.path.join(self.root, self.processed_folder, self.training_file), 'wb') as f:
            torch.save(training_set, f)
        with open(os.path.join(self.root, self.processed_folder, self.test_file), 'wb') as f:
            torch.save(test_set, f)

        print('Done!')

    def __repr__(self):
        fmt_str = 'Dataset ' + self.__class__.__name__ + '\n'
        fmt_str += '    Number of datapoints: {}\n'.format(self.__len__())
        tmp = 'train' if self.train is True else 'test'
        fmt_str += '    Split: {}\n'.format(tmp)
        fmt_str += '    Root Location: {}\n'.format(self.root)
        tmp = '    Transforms (if any): '
        fmt_str += '{0}{1}\n'.format(tmp, self.transform.__repr__().replace('\n', '\n' + ' ' * len(tmp)))
        tmp = '    Target Transforms (if any): '
        fmt_str += '{0}{1}'.format(tmp, self.target_transform.__repr__().replace('\n', '\n' + ' ' * len(tmp)))
        return fmt_str

def get_int(b):
    return int(codecs.encode(b, 'hex'), 16)

def read_label_file(path):
    with open(path, 'rb') as f:
        data = f.read()
        assert get_int(data[:4]) == 2049
        length = get_int(data[4:8])
        parsed = np.frombuffer(data, dtype=np.uint8, offset=8)
        return torch.from_numpy(parsed.copy()).view(length).long()

def read_image_file(path):
    with open(path, 'rb') as f:
        data = f.read()
        assert get_int(data[:4]) == 2051
        length = get_int(data[4:8])
        num_rows = get_int(data[8:12])
        num_cols = get_int(data[12:16])
        images = []
        parsed = np.frombuffer(data, dtype=np.uint8, offset=16)
        return torch.from_numpy(parsed.copy()).view(length, num_rows, num_cols)

# Dataloader for Fashion-MNIST
class F_MNIST_dataloader():
    def __init__(self, batch_size, batch_size_val, noise_type, noise_rate, num_val):
        self.batch_size = batch_size
        self.batch_size_val = batch_size_val
        self.noise_type = noise_type
        self.noise_rate = noise_rate
        self.num_val = num_val

    def run(self):
        train_dataset = F_MNIST(root='./data/',
                                download=True,
                                train=True,
                                transform=transforms.ToTensor(),
                                noise_type=self.noise_type,
                                noise_rate=self.noise_rate
                                )

        test_dataset = F_MNIST(root='./data/',
                               download=True,
                               train=False,
                               transform=transforms.ToTensor(),
                               noise_type=self.noise_type,
                               noise_rate=self.noise_rate
                               )

        val_dataset = F_MNIST(root='./data/',
                              download=True,
                              train=True,
                              transform=transforms.ToTensor(),
                              noise_type='clean',
                              noise_rate=0
                              )

        # random select val data
        val_idx = random.sample(range(len(train_dataset)), self.num_val)
        # remove val data from train data
        data_train = [i for j, i in enumerate(train_dataset) if j not in val_idx]

        data_val = []
        for i in val_idx:
            # prepare val dataset
            data_val.append(val_dataset[i])
            # add val data back to train data
            label = val_dataset[i][1].item()
            list_val = [val_dataset[i][0], label, val_dataset[i][2]]
            data_train.append(list_val)

        train_loader = torch.utils.data.DataLoader(dataset=data_train,
                                                   batch_size=self.batch_size,
                                                   num_workers=0,
                                                   drop_last=True,
                                                   shuffle=True)

        test_loader = torch.utils.data.DataLoader(dataset=test_dataset,
                                                  batch_size=self.batch_size,
                                                  num_workers=0,
                                                  drop_last=True,
                                                  shuffle=False)

        val_loader = torch.utils.data.DataLoader(dataset=data_val,
                                                 batch_size=self.batch_size_val,
                                                 num_workers=0,
                                                 drop_last=True,
                                                 shuffle=True)

        return train_loader, val_loader, test_loader

{'urllib': 'six.moves.urllib', 'gzip': 'gzip'}


#!/usr/bin/python3
"""function that queries the Reddit API and
    returns the number of subscribers"""

import requests

def number_of_subscribers(subreddit):
    """ Reddit API"""
    respons = requests.get("https://www.reddit.com/r/{}/about.json"
                           .format(subreddit), headers={'User-agent': 'hello'})

    if not respons:
        return 0
    return(respons.json().get('data').get('subscribers'))


import cv2
from utils import constants as cs
from utils import utility, os_utils

if __name__ == '__main__':
    fg_bg = cv2.createBackgroundSubtractorMOG2()
    IMAGE_SIZE = (12, 8)

    path_gen = os_utils.iterate_data(cs.BASE_DATA_PATH + cs.DATA_TRAIN_VIDEOS, ".mp4")

    for path in path_gen:
        utility.write_videos(path, cs.DATA_TRAIN_VIDEOS, cs.DATA_BG_TRAIN_VIDEO)

    path_gen = os_utils.iterate_test_data(cs.BASE_DATA_PATH + cs.DATA_TEST_VIDEOS, ".mp4")
    for path in path_gen:
        utility.write_videos(path, cs.DATA_TEST_VIDEOS, cs.DATA_BG_TEST_VIDEO)


'''
@author: Merijn de Bakker, Utrecht University 2016

Module helper contains helper functions used in the mapalgebra module.
'''
import numpy as np
import math
import types
import matplotlib.pyplot as plt
import os
import random

def create_domain(dom, f, phen_name, pset_name, p_name):

    #get a list with item ids
    item_ids = f.item_ids(phen_name, pset_name)

    #check whether the property is a field
    ps_is_field = is_field(f, phen_name, pset_name)

    #iterator counter
    counter = 0

    if ps_is_field:
        spatial_extent = np.asarray(f.get_spatial_extent_pset(phen_name,
                  pset_name))

        for i in item_ids:
            dom.set_as_current_domain_for_id(i, spatial_extent)
            counter += 1

    else:
        locations = f.get_locations(phen_name, pset_name, p_name)

        for i in item_ids:
            [locations[counter][0], locations[counter][1]]
            dom.set_as_current_domain_for_id(i, np.asarray(
                       [locations[counter][0], locations[counter][1]]))
            counter += 1

    return dom

def create_value(v, f, phen_name, pset_name, p_name):

    # get a list with item ids
    item_ids = f.item_ids(phen_name, pset_name)

    # iterator counter
    counter = 0

    retrieved_data = f.get_property_value(phen_name, pset_name, p_name).astype(
               float)

    if is_field(f, phen_name, pset_name):
        for i in item_ids:
            v.set_as_current_value_for_id(i, retrieved_data)
            counter += 1
    else:
        for i in item_ids:
            v.set_as_current_value_for_id(i, np.asarray(
                        [retrieved_data[counter]]))
            counter += 1

    return v

def get_datatype(val):
    if val.get_current_value_for_id(0).ndim >= 2:
        return datatypes.field
    else:
        return datatypes.agent

def is_field(f, phen_name, pset_name):

    p_list = f.list_property_sets_prop(phen_name, pset_name)

    return f.is_field(phen_name, pset_name, p_list[0])

def get_cell_at_loc(cells_domain, cells_discretization, location):
    x_min, y_min, x_max, y_max = cells_domain
    x_coord = location[0]
    y_coord = location[1]
    west = x_min
    north = y_max
    cell_size = get_cellsize(x_max, x_min, cells_discretization[0])

    x_col = (x_coord - west) / cell_size
    y_row = (north - y_coord) / cell_size

    row_idx = int(math.floor(y_row))
    col_idx = int(math.floor(x_col))

    nr_rows = int(math.floor((y_max - y_min) / cell_size))
    nr_cols = int(math.floor((x_max - x_min) / cell_size))

    if (col_idx + 1) > nr_cols or (col_idx) <= 0:
        msg = "Column index ({0}) out of range (from 1 to {1}))".format(
                    col_idx + 1, nr_cols)
        raise ValueError(msg)

    if (row_idx + 1) > nr_rows or (row_idx) <= 0:
        msg = "Row index ({0}) out of range (from 1 to {1})".format(
                    row_idx + 1, nr_rows)
        raise ValueError(msg)

    return row_idx, col_idx

def get_cell_value_at_location(cells, cells_domain, cells_discretization,
                               location):

    row_idx, col_idx = get_cell_at_loc(cells_domain, cells_discretization,
                                       location)

    return [row_idx, col_idx, cells[row_idx, col_idx]]

def get_new_random_loc(current_x, current_y, sigma):
    return [random.normalvariate(current_x, sigma),
                    random.normalvariate(current_y, sigma)]

# assumes square cells
def get_cellsize(x_max, x_min, nr_cols):

    return (x_max - x_min) / nr_cols

def create3Darray(data):
    if data.ndim == 1:
        data = np.expand_dims(data, 2)

    return np.expand_dims(data, 3)

class datatypes:
    scalar_types = (types.IntType, types.LongType, types.FloatType,
                    types.ListType, np.ndarray)
    field, agent = range(2)

    @staticmethod
    def get_type_string(phenomenon):
        if phenomenon == datatypes.field:
            return 'Raster'
        if phenomenon == datatypes.agent:
            return 'Agent'
        else:
            return ''

def transform(values):
    nr_rows = 100  # 6
    nr_cols = 100  # 4
    cellsize = 25
    """ Transforms x,y PCRaster coordindates to plot canvas coordinates """
    xfac = nr_cols / float(nr_cols * cellsize)
    yfac = nr_rows / float(nr_rows * cellsize)

    x = values[0] * xfac - 0.5
    y = -1.0 * values[1] * yfac - 0.5

    return x, y

def make_png(field, agents, timestep):
    plt.rcParams.update({'font.size': 22})
    plt.figure(figsize=(8, 8), dpi=80)
    plt.imshow(field, interpolation="none",
               extent=[0, 100, 100, 0], origin='upper',
               cmap=plt.get_cmap("Greens"), label='biomass')
    plt.title("timestep {0}".format(timestep))

    plt.colorbar(shrink=0.7)

    locs = []
    for a in agents:
        locs.append(transform(a))
    x, y = zip(*locs)

    plt.plot(x, y, 'ro', color='red', markersize=4, label='cows')
    plt.ylim(len(field), 0)

    # min/max biomass values...
    plt.clim(0, 20)

    plt.xticks([])
    plt.yticks([])

    plt.legend(loc='upper right')

    plt.savefig(os.path.join("../data/model_t{:05d}.png".format(timestep)),
                transparent=False, bbox_inches='tight')
    plt.close()


"""
    基于 selenium.webdriver 操控Chrome浏览器搜索信息，并提取第一个标题信息
    注1：需要安装 selenium: pip install selenium
    注2：需要下载 chromedriver.exe（版本要与实际浏览器一致），放置到 python3安装根目录下
    注3：selenium的性质是一个web测试工具，用于模拟人操作浏览器用的
"""
import time
from selenium import webdriver

keys = "lifeeeee"
# 创建Chrome驱动器
driver = webdriver.Chrome()
# 打开网页
driver.get("https://www.baidu.com/")
time.sleep(1)
# 定位到搜索栏，并在搜索栏输入关键字
inputElement = driver.find_element_by_name("wd")
inputElement.send_keys(keys)
# 定位到提交按钮，点击提交
su_find = driver.find_element_by_id('su')
su_find.click()
time.sleep(3)
# 搜索name为 description的标签
re_title = driver.find_element_by_name('')
print(re_title.get_attribute("content"))
# 退出
time.sleep(3)
driver.quit()


import pandas as pd
from NaiveBayesQuestionClassifier import NBFunctions
from sklearn.feature_extraction.text import CountVectorizer
from nltk.tokenize import word_tokenize

print("Reading Training data ... ")
questionFile = pd.read_csv("data/filtratedTrainingQuestions.csv")

# --> Helpers
# print(questionFile.iloc[0])
# print(questionFile.loc[:,'Category'])
# print(questionFile.loc[questionFile.Category == "Logic"])
# questionFile['test']='success' # assigning to the table
# <-- End.

def wordProbability(wordList, countList):
    probWords = []
    for eachWord, count in zip(wordList, countList):
        probWords.append(count/len(wordList))
    return(dict(zip(wordList, probWords)))

print("Training NB algo")
questionDocs = questionFile.loc[:,'Question']
vectorizer = CountVectorizer()
X = vectorizer.fit_transform(questionDocs)
totalFeatures = len(vectorizer.get_feature_names())

# # --> CountVectorizer for logic questions
logicDocs = questionFile.loc[questionFile.Category == "Logic",'Question']
logicVectorizer = CountVectorizer()
X_logic = logicVectorizer.fit_transform(logicDocs)
word_list_logic = logicVectorizer.get_feature_names();
count_list_logic = X_logic.toarray().sum(axis=0)
freqLogic = dict(zip(word_list_logic,count_list_logic))
logicWordsProb = wordProbability(word_list_logic, count_list_logic)
totalCntLogic = count_list_logic.sum(axis=0)
# print(logicWordsProb)
# # --> End.

# # --> CountVectorizer for fraction questions
fractionDocs = questionFile.loc[questionFile.Category == "Fractions",'Question']
fractionVectorizer = CountVectorizer()
X_fraction = fractionVectorizer.fit_transform(fractionDocs)
word_list_fraction = fractionVectorizer.get_feature_names();
count_list_fraction = X_fraction.toarray().sum(axis=0)
freqFraction = dict(zip(word_list_fraction,count_list_fraction))
fractionWordsProb = wordProbability(word_list_fraction, count_list_fraction)
totalCntFraction = count_list_fraction.sum(axis=0)
# print(fractionWordsProb)
# # --> End.

# # --> CountVectorizer for algebra questions
algebraDocs = questionFile.loc[questionFile.Category == "Algebra",'Question']
algebraVectorizer = CountVectorizer()
X_algebra = algebraVectorizer.fit_transform(algebraDocs)
wordListAlgebra = algebraVectorizer.get_feature_names();
countListAlgebra = X_algebra.toarray().sum(axis=0)
freqAlgebra = dict(zip(wordListAlgebra,countListAlgebra))
algebraWordsProb = wordProbability(wordListAlgebra, countListAlgebra)
totalCntAlgebra = countListAlgebra.sum(axis=0)
# # --> End.

# # --> CountVectorizer for comparison questions
comparisonDocs = questionFile.loc[questionFile.Category == "Comparisons",'Question']
comparisonVectorizer = CountVectorizer()
X_comparison = comparisonVectorizer.fit_transform(comparisonDocs)
wordListComparison = comparisonVectorizer.get_feature_names();
countListComparison = X_comparison.toarray().sum(axis=0)
freqComparison = dict(zip(wordListComparison,countListComparison))
comparisonWordsProb = wordProbability(wordListComparison, countListComparison)
totalCntComparison = countListComparison.sum(axis=0)
# # --> End.

# --> CountVectorizer for percentage questions
percentageDocs = questionFile.loc[questionFile.Category == "Percentages",'Question']
# print(percentageDocs)
percentageVectorizer = CountVectorizer()
X_percentage = percentageVectorizer.fit_transform(percentageDocs)
wordListPercentage = percentageVectorizer.get_feature_names();
countListPercentage = X_percentage.toarray().sum(axis=0)
freqPercentage = dict(zip(wordListPercentage,countListPercentage))
percentageWordsProb = wordProbability(wordListPercentage, countListPercentage)
totalCntPercentage = countListPercentage.sum(axis=0)
# --> End.

# # --> CountVectorizer for probability questions
probabilityDocs = questionFile.loc[questionFile.Category == "Probability",'Question']
probabilityVectorizer = CountVectorizer()
X_probability = probabilityVectorizer.fit_transform(probabilityDocs)
wordListProbability = probabilityVectorizer.get_feature_names();
countListProbability = X_probability.toarray().sum(axis=0)
freqProbability = dict(zip(wordListProbability,countListProbability))
probabilityWordsProb = wordProbability(wordListProbability, countListProbability)
totalCntProbability = countListProbability.sum(axis=0)
# # --> End.

# --> CountVectorizer for general knowledge questions
gkDocs = questionFile.loc[questionFile.Category == "General Knowledge",'Question']
gkVectorizer = CountVectorizer()
X_gk = gkVectorizer.fit_transform(gkDocs)
wordListGk = gkVectorizer.get_feature_names()
countListGk = X_gk.toarray().sum(axis=0)
freqGk = dict(zip(wordListGk,countListGk))
gkWordsProb = wordProbability(wordListGk, countListGk)
totalCntGk = countListGk.sum(axis=0)
# --> End.

# <-- Naive Bayes Function
categoryList = ['Logic','Fraction','Algebra','Comparison','Percentage','Probability','Gk']

def calcProbability(question):
    """
    Function to calcuate probability of question being in a certain category
    :param question: question to calculate probability of
    :return: maxProbCategory category with the highest probability.
    """
    # print("Classifying Question:",question)
    question = NBFunctions.stopWordRemoval(question)
    wordList = word_tokenize(question)
    wordProbabilities = []
    for eachCategory in categoryList:
        if eachCategory == "Logic":
            probWordInCategory = NBFunctions.bayesWordProb(wordList, freqLogic, totalCntLogic, totalFeatures)
        elif eachCategory == "Fraction":
            probWordInCategory = NBFunctions.bayesWordProb(wordList, freqFraction, totalCntFraction, totalFeatures)
        elif eachCategory == "Algebra":
            probWordInCategory = NBFunctions.bayesWordProb(wordList, freqAlgebra, totalCntAlgebra, totalFeatures)
        elif eachCategory == "Comparison":
            probWordInCategory = NBFunctions.bayesWordProb(wordList, freqComparison, totalCntComparison, totalFeatures)
        elif eachCategory == "Percentage":
            probWordInCategory = NBFunctions.bayesWordProb(wordList, freqPercentage, totalCntPercentage, totalFeatures)
        elif eachCategory == "Probability":
            probWordInCategory = NBFunctions.bayesWordProb(wordList, freqProbability, totalCntProbability, totalFeatures)
        elif eachCategory=="Gk":
            probWordInCategory = NBFunctions.bayesWordProb(wordList, freqGk, totalCntGk, totalFeatures)
        wordProbabilities.append(probWordInCategory)

    probabilityMap = {}
    probabilityMap = NBFunctions.probProduct(categoryList, wordProbabilities, probabilityMap)
    maxProbCategory = max(probabilityMap, key=probabilityMap.get)
    return (maxProbCategory)
# --> End.
trialQuestion = "Write the next term of the series below: 1, 1, 2, 4, 3, 9, 4 _ _ _"
print(calcProbability(trialQuestion))
# wordList = word_tokenize(question)R
# NBFunctions.bayesWordProb(wordList,freqGk, totalCntGk,totalFeatures)
# NBFunctions.bayesWordProb(wordList,freqLogic, totalCntGk,totalFeatures)


# Create your views here.
from django.contrib.auth.decorators import login_required
from django.contrib.auth.models import User
from django.core.urlresolvers import reverse
from django.db.models import Q
from django.http import HttpResponse, HttpResponseForbidden, HttpResponseRedirect, Http404
from django.shortcuts import get_object_or_404, render_to_response
from django.template import RequestContext
from django.utils.encoding import smart_str
from genenews_main.models import Article, Gene, Sequence, Vote, LeaderboardCache, get_user_score, type_to_date
from genenews_main.forms import SubmissionForm

def index(request):
    articles = Article.objects.all().order_by('-pk')[:25]
    entries = add_votes(request, articles)
    return render_to_response('index.html',{'entries': entries}, context_instance=RequestContext(request))

def leaderboard(request):
    entries = []
    type = request.GET.get('type', 'all')

    if type not in ['all', 'year', 'month', 'day']:
        type = 'all'

    LeaderboardCache.objects.update(type=type) # only temporarily until a cron job can be set up
    for entry in LeaderboardCache.objects.filter(type=type):
        entries.append((entry.score, entry.user, entry.user.article_set.filter(date__gte=type_to_date(type)).count()))
    entries.sort()
    entries.reverse()
    return render_to_response('leaderboard.html', {'entries': entries, 'type': type}, context_instance=RequestContext(request))

@login_required
def submit(request):
    genes = []
    if request.method=="POST":
        form = SubmissionForm(request.POST)
        try:
            genes = [Gene.objects.get(pk=long(gene_id)) for gene_id in request.POST.getlist('gene')]
        except Gene.DoesNotExist:
            return HttpResponseForbidden()
        if form.is_valid():
            article = form.save(commit=False)
            article.user = request.user
            article.save()
            map(lambda g: article.genes.add(g), genes)
            article.save()
            v = Vote(article=article, user=request.user, vote=1)
            v.save()
            return HttpResponseRedirect(reverse('article', args=[article.id]))
    else:
        form = SubmissionForm()
    return render_to_response('submit.html', {'form': form, 'genes': genes}, context_instance=RequestContext(request))

@login_required
def gene_autocomplete(request):
    if request.method == "GET":
        if not request.GET.has_key('q') or not request.GET.has_key('limit'):
            return HttpResponseForbidden()
        try:
            int(request.GET.get('limit'))
        except ValueError:
            return HttpResponseForbidden()
        qtemp = Q(name__icontains=request.GET.get('q'))
        genes = Gene.objects.filter(qtemp)[:request.GET.get('limit', 10)]
        tempstr = '['
        for index, m in enumerate(genes):
            temp = {}
            temp['id'] = smart_str(m.id)
            temp['name'] = smart_str(m.medname())
            if index > 0:
                tempstr += ','
            tempstr += str(temp)
        tempstr += ']'
        return HttpResponse(tempstr, mimetype="application/json")
    resp = HttpResponse()
    resp.status_code = 405
    return resp

def gene_page(request, gene_name):
    genes = Gene.objects.filter(name=gene_name)
    if not genes.exists():
        raise Http404
    gene = genes[0] # just get the first gene that comes up

    articles = gene.article_set.all()
    entries = add_votes(request, articles)
    seq = gene.sequence
    return render_to_response('gene_page.html', {'gene': gene, 'entries': entries, 'seq':seq}, context_instance=RequestContext(request))

def add_votes(request, articles):
    if request.user.is_anonymous():
        return [(article, None) for article in articles]

    else:
        out = []
        for article in articles:
            if article.vote_set.filter(user=request.user).exists():
                out.append((article, article.vote_set.get(user=request.user)))
            else:
                out.append((article, None))

        return out


from django.urls import path
from .views import checkout, orders, order_detail, order_confirm

urlpatterns = [
    path('checkout/', checkout, name='checkout'),
    path('all/', orders, name='orders'),
    path('order_detail/', order_detail, name='order_detail'),
    path('OrderConfirmed/', order_confirm, name='order_confirmed'),
]


from Models.clustering import KMeans
from datasets import get_dataset
from Pipelines.clustering_pipeline import get_clustering_res
from Metrics.metrics import accuracy
import matplotlib.pyplot as plt

def get_color(l):

    if l == 0:
        return "red"
    elif l == 1:
        return "blue"
    elif l == 2:
        return "yellow"

def plot_clustering(x, y, title):

    fig = plt.figure(figsize=(x.shape[1] * 5, x.shape[1] * 5))

    plt.title(title)
    for i in range(x.shape[1]):
        for j in range(x.shape[1]):

            plt.subplot(x.shape[1], x.shape[1], 1 + i * x.shape[1] + j)
            plt.scatter(x[:, i], x[:, j], c = [get_color(lab) for lab in y])
    plt.show()

def clustering_test():

    cluster = KMeans(k=3)

    x, y = get_dataset("iris")

    outp = get_clustering_res({"kmeans": cluster}, x)

    plot_clustering(x, y, "original")
    plot_clustering(x, outp[0], "clustering")

if __name__ == "__main__":

    clustering_test()


import matplotlib.pyplot as plt
from matplotlib import gridspec

import settings
from gtfspy.routing.node_profile_analyzer_time_and_veh_legs import NodeProfileAnalyzerTimeAndVehLegs
from gtfspy.routing.multi_objective_pseudo_connection_scan_profiler import MultiObjectivePseudoCSAProfiler
from gtfspy.routing.connection import Connection

import networkx
import matplotlib as mpl
mpl.style.use('classic')

from matplotlib import rc
# rc('font',**{'family':'sans-serif','sans-serif':['Helvetica']})
## for Palatino and other serif fonts use:
#rc('font',**{'family':'serif','serif':['Palatino']})
rc('text', usetex=True)

OFFSET = 8*3600
events = [
    ("a", "b", 0, 12, "bus_1"),
    ("a", "b", 20, 32, "bus_1"),
    ("a", "b", 40, 52, "bus_1"),
    ("c", "d", 10, 12, "tram_1"),
    ("c", "d", 30, 32, "tram_1"),
    ("d", "e", 15, 17, "bus_2"),
    ("d", "e", 40, 42, "bus_2"),
    ("f", "g", 22, 24, "tram_2"),
    ("f", "g", 42, 44, "tram_2"),
    ("h", "i", 24, 27, "rail"),
    ("j", "k", 00,  6, "subway_1"),
    ("j", "k", 30, 36, "subway_1"),
]
events = [(e[0], e[1], e[2]* 60 + OFFSET, e[3] * 60 + OFFSET, e[4]) for e in events]
nodes = "OabcdefghijkD"
node_name_to_index = {name:i for i, name in enumerate(nodes)}
max_y = 1
min_y = 0
max_t = max((event[3] for event in events))
min_t = min((event[2] for event in events))

dummy_seq = 1
connections = [Connection(e[0], e[1], e[2], e[3], e[4], dummy_seq, is_walk=False)
               for e in events]

g = networkx.Graph()
g.add_edge("O", "a", d_walk=7)
g.add_edge("O", "c", d_walk=2)
g.add_edge("O", "h", d_walk=5)
g.add_edge("b", "D", d_walk=18)
g.add_edge("d", "i", d_walk=3)
g.add_edge("e", "f", d_walk=3)
g.add_edge("e", "D", d_walk=15)
g.add_edge("g", "D", d_walk=4)
g.add_edge("i", "j", d_walk=2)
g.add_edge("k", "D", d_walk=3)
g.add_edge("O", "D", d_walk=60)

profiler = MultiObjectivePseudoCSAProfiler(connections, "D", min_t, max_t, transfer_margin=0, walk_network=g, walk_speed=1.0/60.0)
pseudo_connections = profiler._pseudo_connections

def _t_to_x(t):
    assert(t >= min_t)
    assert(t <= max_t)
    return 0.15 + 0.75 * (t - min_t) / (max_t - min_t)

n_nodes = len(node_name_to_index)
y_offset_top = 0.05
y_offset_down = 0.1

def plot_temporal_network():
    fig = plt.figure(figsize=(8, 4))
    ax = fig.add_subplot(111)

    step = (max_y - min_y - y_offset_down - y_offset_top)/(n_nodes)
    node_ys = [step * (n_nodes - i  - 0.5 ) + y_offset_down for i in range(n_nodes)]
    for y, letter in zip(node_ys, nodes):
        ax.plot([0.1, 0.95], [y, y], color="black", ls="-", lw=2)
        ax.text(0.08, y, letter, va="center", ha="right")

    from gtfspy.route_types import ROUTE_TYPE_TO_COLOR, ROUTE_TYPE_TO_LOWERCASE_TAG, WALK
    trip_id_to_color = {}
    for event in events:
        trip_id = event[-1]
        for route_type, tag in ROUTE_TYPE_TO_LOWERCASE_TAG.items():
            if tag in trip_id:
                trip_id_to_color[trip_id] = ROUTE_TYPE_TO_COLOR[route_type]
    trip_id_to_color['walk'] = ROUTE_TYPE_TO_COLOR[WALK]

    for i, pseudo_connection in enumerate(pseudo_connections):
        arrival_stop = pseudo_connection.arrival_stop
        arrival_stop_next_departure_time = pseudo_connection.arrival_stop
        arr_time = pseudo_connection.arrival_time
        departure_stop = pseudo_connection.departure_stop
        dep_time = pseudo_connection.departure_time
        color = trip_id_to_color["walk"]
        xs = [_t_to_x(t) for t in [dep_time, arr_time]]
        origin_y = node_ys[node_name_to_index[departure_stop]]
        destination_y = node_ys[node_name_to_index[arrival_stop]]
        from_xy = [xs[0], origin_y]
        to_xy = [xs[1], destination_y]
        ax.annotate("",
                     xy=to_xy,
                     xytext=from_xy,
                     arrowprops=dict(arrowstyle="simple", fc="white", ec=color, lw=1.5, linestyle="-"),
                     xycoords="data",
                     textcoords="data")

    for origin, destination, dep_time, arr_time, name in events:
        color = trip_id_to_color[name]
        origin_index = node_name_to_index[origin]
        origin_y = node_ys[origin_index]
        destination_index = node_name_to_index[destination]
        destination_y = node_ys[destination_index]
        xs = [_t_to_x(t) for t in [dep_time, arr_time]]
        from_xy = [xs[0], origin_y]
        to_xy = [xs[1], destination_y]
        print(from_xy)
        print(to_xy)
        ax.annotate("",
                     xy=to_xy,
                     xytext=from_xy,
                     arrowprops=dict(arrowstyle="simple", color=color, ls="-"),
                     xycoords="data",
                     textcoords="data")

    ax.set_xlim(0, 1)
    ax.set_ylim(0, 1)

profiler.run()
profile = profiler.stop_profiles['O']

def plot_plain_profile(profile):
    gs = gridspec.GridSpec(1, 6)
    gs.update(left=0.1, right=0.96, wspace=0.2, bottom=0.18)
    analyzer = NodeProfileAnalyzerTimeAndVehLegs(profile, 8*3600, 8.5*3600).get_time_profile_analyzer()
    fig = plt.figure(figsize=(5.5, 3.5))

    ax1 = plt.subplot(gs[:, :4])
    analyzer.plot_temporal_distance_profile(format_string="%H:%M",
                                            plot_journeys=True,
                                            journey_letters="BFH",
                                            lw=3,
                                            ax=ax1,
                                            plot_tdist_stats=True,
                                            alpha=0.15,
                                            plot_trip_stats=False,
                                            duration_divider=60)
    print("plotted")
    ax2 = plt.subplot(gs[:, 4:])

    ax2.set_xlim(0, 0.1)

    fig = analyzer.plot_temporal_distance_pdf_horizontal(use_minutes=True,
                                                         ax=ax2,
                                                         legend_font_size=9,
                                                         legend_loc="lower right")

    ax2.set_ylabel("")
    ax2.set_yticks([])
    ax2.set_yticklabels(["" for _ in ax2.get_yticks()])

    ax2.set_xticks([0.05, 0.10])
    ax2.set_yticklabels(["0.05", "0.10"])

    ax1.set_ylim(0, 67)
    ax2.set_ylim(0, 67)

    ax1.set_xlabel("Departure time $t_{\\text{dep}}$ (min)")
    ax1.set_ylabel("Temporal distance $\\tau$ (min)")

    handles, labels = ax1.get_legend_handles_labels()

    ax1.legend(handles, labels, loc="best", ncol=2, shadow=False, prop={'size': 9})
    for _ax, letter in zip([ax1, ax2], "AB"):
        _ax.text(0.04, 0.98,
                  "\\textbf{" + letter + "}",
                  horizontalalignment="left",
                  verticalalignment="top",
                  transform=_ax.transAxes,
                  fontsize=15,
                  color="black")

    fig.savefig(settings.FIGS_DIRECTORY + "schematic_temporal_distance.pdf")

def plot_transfer_profile(profile):
    gs = gridspec.GridSpec(1, 6)
    gs.update(left=0.1, right=0.96, wspace=0.15, bottom=0.18)
    analyzer = NodeProfileAnalyzerTimeAndVehLegs(profile, 8*3600, 8*3600 + 30 * 60)
    print(analyzer.mean_n_boardings_on_shortest_paths())

    fig = plt.figure(figsize=(5.5, 3.5))
    ax1 = plt.subplot(gs[:, :4])
    fig = analyzer.plot_new_transfer_temporal_distance_profile(format_string="%H:%M",
                                                               duration_divider=60,
                                                               default_lw=4,
                                                               journey_letters="BCEFH",
                                                               plot_journeys=True,
                                                               ax=ax1,
                                                               ncol_legend=2,
                                                               legend_font_size=9)

    ax1.set_xlabel("Departure time $t_{\\text{dep}}$ (min)")
    ax1.set_ylabel("Temporal distance $\\tau$ (min)")

    ax2 = plt.subplot(gs[:, 4:])
    ax2 = analyzer.plot_temporal_distance_pdf_horizontal(use_minutes=False,
                                                         duration_divider=60,
                                                         ax=ax2,
                                                         legend_font_size=9,
                                                         legend_loc="lower right")

    ax2.set_ylabel("")

    ax1.set_ylim(0, 67)
    ax2.set_ylim(0, 67)
    ax2.set_xlim(0, 0.1)
    ax2.set_yticklabels(["" for _ in ax2.get_yticks()])
    ax2.set_xlabel("Probability density $P(\\tau)$")
    ax2.set_xticks([0.05, 0.1])
    ax2.get_yaxis().set_tick_params(direction='in')

    for _ax, letter in zip([ax1, ax2], "AB"):
        _ax.text(0.04, 0.98,
                  "\\textbf{" + letter + "}",
                  horizontalalignment="left",
                  verticalalignment="top",
                  transform=_ax.transAxes,
                  fontsize=15,
                  color="black",
                  zorder=99999999)
    fig.savefig(settings.FIGS_DIRECTORY + "schematic_transfer_profile.pdf")

# plot_temporal_network()
plot_plain_profile(profile)
plot_transfer_profile(profile)
# plt.show()

plt.show()


from django.urls import path
from django.contrib.auth import views as auth_views
from django.views.generic import DetailView
from . import views

urlpatterns = [
    path('', views.Home, name='garage-home'),
    path('about/', views.About, name='garage-about'),
    path('service/', views.Service, name='garage-service'),
    path('history/<int:pk>', views.ServiceDetailView.as_view(), name='garage-history-detail'),
    path('print/<int:pk>', views.GeneratePDF.as_view(), name='print-pdf'),
    path('history/', views.History, name='garage-history'),
    path('login/', auth_views.LoginView.as_view(template_name='login.html'), name='garage-login'),
    path('logout/', auth_views.LogoutView.as_view(template_name='home.html'), name='garage-logout'),
    path('register/', views.Register, name='garage-register'),
    path('jobs/', views.JobListView.as_view(), name='garage-jobs'),
    path('jobs/<int:pk>', views.JobUpdateView.as_view(), name='garage-jobs-update'),
]


from copy import copy
from typing import Optional

from common.constants import *

def get_document(id: str) -> Optional[dict]:
    return HTTP_CLIENT.get(
        f"{API_BASE_URL}/v1/documents/{id}",
        headers=API_HEADERS,
        params=API_BASE_PARAMS,
    )

def get_document_previews(id: str) -> Optional[dict]:
    return HTTP_CLIENT.get(
        f"{API_BASE_URL}/v1/documents/{id}/previewInfo",
        headers=API_HEADERS,
        params=API_BASE_PARAMS,
    )

def get_search(query: str) -> Optional[dict]:
    data = {
        "client": "ios",
        "client_id": API_CLIENT_ID,
        "client_secret": API_CLIENT_SECRET,
        "filters[type][]": ["document"],
        "limit": "20",
        "location[auto]": "1",
        "offset": "0",
        "query": query,
        "sort": "relevancy",
        "trigger": "ios_app",
        "view": "list_m",
    }
    response = HTTP_CLIENT.post(
        f"{API_BASE_URL}/v2/search/",
        headers=API_HEADERS,
        data=data,
    )
    if response.status_code == 200:
        return response.json()

def get_search_suggestions(query: str) -> Optional[dict]:
    params = copy(API_BASE_PARAMS)
    params["query"] = query
    response = HTTP_CLIENT.get(
        f"{API_BASE_URL}/v1/search/suggest",
        headers=API_HEADERS,
        params=params,
    )
    if response.status_code == 200:
        return response.json()


"""
"""
import os
import GalaxyTestBase
import test_util
from bioblend.galaxy.tools.inputs import (
    inputs,
    dataset,
    repeat,
    conditional,
)

def get_abspath(path):
    return os.path.join(os.path.dirname(__file__), path)

@test_util.skip_unless_galaxy()
class TestGalaxyTools(GalaxyTestBase.GalaxyTestBase):

    def setUp(self):
        super(TestGalaxyTools, self).setUp()

    def test_get_tools(self):
        # Test requires target Galaxy is configured with at least one tool.
        tools = self.gi.tools.get_tools()
        assert len(tools) > 0
        assert all(map(self._assert_is_tool_rep, tools))

    def test_get_tool_panel(self):
        # Test requires target Galaxy is configured with at least one tool
        # section.
        tool_panel = self.gi.tools.get_tool_panel()
        sections = [s for s in tool_panel if "elems" in s]
        assert len(sections) > 0
        assert all(map(self._assert_is_tool_rep, sections[0]["elems"]))

    def _assert_is_tool_rep(self, data):
        assert data["model_class"].endswith("Tool")
        # Special tools like SetMetadataTool may have different model_class
        # than Tool - but they all seem to end in tool.

        for key in ["name", "id", "version"]:
            assert key in data, "key %s not in %s" % (key, data)
        return True

    def test_paste_data(self):
        history = self.gi.histories.create_history(name="test_paste_data history")

        tool_output = self.gi.tools.paste_content("test contents", history["id"])
        assert len(tool_output["outputs"]) == 1

    def test_upload_file(self):
        history = self.gi.histories.create_history(name="test_upload_file history")

        fn = get_abspath(os.path.join(os.pardir, "setup.py"))
        file_name = "test1"
        tool_output = self.gi.tools.upload_file(
            fn,
            # First param could be a regular path also of course...
            history_id=history["id"],
            file_name=file_name,
            dbkey="?",
            file_type="txt",
        )
        assert len(tool_output["outputs"]) == 1
        assert tool_output['outputs'][0]['name'] == file_name

    @test_util.skip_unless_tool("random_lines1")
    def test_run_random_lines(self):
        # Run second test case from randomlines.xml
        history_id = self.gi.histories.create_history(name="test_run_random_lines history")["id"]
        with open(get_abspath(os.path.join("data", "1.bed"))) as f:
            contents = f.read()
        dataset_id = self._test_dataset(history_id, contents=contents)
        tool_inputs = inputs().set(
            "num_lines", "1"
        ).set(
            "input", dataset(dataset_id)
        ).set(
            "seed_source", conditional().set(
                "seed_source_selector", "set_seed"
            ).set(
                "seed", "asdf"
            )
        )
        tool_output = self.gi.tools.run_tool(
            history_id=history_id,
            tool_id="random_lines1",
            tool_inputs=tool_inputs
        )
        assert len(tool_output["outputs"]) == 1
        # TODO: Wait for results and verify has 1 line and is
        # chr5  131424298   131424460   CCDS4149.1_cds_0_0_chr5_131424299_f 0   +

    @test_util.skip_unless_tool("cat1")
    def test_run_cat1(self):
        history_id = self.gi.histories.create_history(name="test_run_cat1 history")["id"]
        dataset1_id = self._test_dataset(history_id, contents="1 2 3")
        dataset2_id = self._test_dataset(history_id, contents="4 5 6")
        dataset3_id = self._test_dataset(history_id, contents="7 8 9")
        tool_inputs = inputs().set(
            "input1", dataset(dataset1_id)
        ).set(
            "queries", repeat().instance(
                inputs().set("input2", dataset(dataset2_id))
            ).instance(
                inputs().set("input2", dataset(dataset3_id))
            )
        )
        tool_output = self.gi.tools.run_tool(
            history_id=history_id,
            tool_id="cat1",
            tool_inputs=tool_inputs
        )
        assert len(tool_output["outputs"]) == 1
        # TODO: Wait for results and verify it has 3 lines - 1 2 3, 4 5 6,
        # and 7 8 9.


import functools
import logging

from ...archive import BaseArchive, metadata

CONTENT_ORDER = ['-date(updated)', '-views']

def multiarg(query, n):
    """ Returns version of query where '??' is replaced by n placeholders """
    return query.replace('??', ', '.join(['%s'] * n))

class AttrDict(dict):

    def __init__(self, *args, **kwargs):
        super(AttrDict, self).__init__(*args, **kwargs)
        self.__dict__ = self

def row_to_dict(row):
    return AttrDict((key, row[key]) for key in row.keys())

def to_dict(func):
    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        row = func(*args, **kwargs)
        if not row:
            return row
        return row_to_dict(row)
    return wrapper

def to_dict_list(func):
    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        rows = func(*args, **kwargs)
        if not rows:
            return rows
        return map(row_to_dict, rows)
    return wrapper

class Transformation(object):
    pass

class Merge(Transformation):
    """Merge into parent"""
    pass

class Ignore(Transformation):
    """Ignores / deletes key and value"""
    pass

class Rename(Transformation):
    """Renames key to specified name"""
    def __init__(self, name):
        self.name = name

class EmbeddedArchive(BaseArchive):
    transformations = [
        {'content': Merge},
        {'replaces': Ignore},
        {'archive': Ignore},
        {'url': Ignore},
        {'video': [
            {'size': Rename('resolution')}
        ]},
        {'image': [
            {'size': Rename('resolution')}
        ]}
    ]
    schema = {
        'content': {
            'constraints': ['path']
        },
        'generic': {
            'constraints': ['path']
        },
        'html': {
            'constraints': ['path']
        },
        'video': {
            'constraints': ['path']
        },
        'audio': {
            'relations': {'many': ['playlist']},
            'constraints': ['path']
        },
        'app': {
            'constraints': ['path']
        },
        'image': {
            'relations': {'many': ['album']},
            'constraints': ['path']
        },
        'playlist': {
            'constraints': ['path', 'file']
        },
        'album': {
            'constraints': ['path', 'file']
        }
    }

    @to_dict
    def one(self, *args, **kwargs):
        return self.db.fetchone(*args, **kwargs)

    @to_dict_list
    def many(self, *args, **kwargs):
        return self.db.fetchall(*args, **kwargs)

    def __init__(self, fsal, db, **config):
        self.db = db
        super(EmbeddedArchive, self).__init__(fsal, **config)

    def _serialize(self, metadata, transformations):
        for transformer in transformations:
            ((key, action),) = transformer.items()
            if isinstance(action, list) and key in metadata:
                self._serialize(metadata[key], action)
            elif action is Merge:
                value = metadata.pop(key, None)
                if value is not None:
                    metadata.update(value)
            elif action is Ignore:
                metadata.pop(key, None)
            elif isinstance(action, Rename):
                value = metadata.pop(key, None)
                if value is not None:
                    metadata[action.name] = value

    def _add_filters(self, q, terms, lang, content_type):
        if lang:
            q.where += 'language = %(lang)s'

        if terms:
            q.where += ('title ILIKE %(terms)s OR '
                        'publisher ILIKE %(terms)s OR '
                        'keywords ILIKE %(terms)s')

        if content_type:
            # get integer representation of content type
            content_type_id = metadata.CONTENT_TYPES[content_type]
            q.where += '("content_type" & %(content_type)s) = %(content_type)s'
        else:
            # exclude content types that cannot be displayed on the mixed type
            # content list
            content_type_id = sum([metadata.CONTENT_TYPES[name]
                                   for name in self.exclude_from_content_list])
            qs = '("content_type" & %(content_type)s) != %(content_type)s'
            q.where += qs

        return (q, content_type_id)

    def get_count(self, terms=None, lang=None, content_type=None):
        q = self.db.Select('COUNT(*) as count',
                           sets='content',
                           where='disabled = false')
        (q, content_type_id) = self._add_filters(q,
                                                 terms,
                                                 lang,
                                                 content_type)
        terms = '%' + terms.lower() + '%'
        result = self.db.fetchone(q, dict(terms=terms,
                                          lang=lang,
                                          content_type=content_type_id))
        return result['count']

    def get_content(self, terms=None, offset=0, limit=0, lang=None,
                    content_type=None):
        # TODO: tests
        q = self.db.Select(sets='content',
                           where='disabled = false',
                           order=CONTENT_ORDER,
                           limit=limit,
                           offset=offset)
        (q, content_type_id) = self._add_filters(q,
                                                 terms,
                                                 lang,
                                                 content_type)
        terms = '%' + terms.lower() + '%'
        results = self.many(q, dict(terms=terms,
                                    lang=lang,
                                    content_type=content_type_id))
        if results and content_type in self.prefetchable_types:
            for meta in results:
                self._fetch(content_type, meta['path'], meta)

        return results

    def _fetch(self, table, relpath, dest, many=False):
        q = self.db.Select(sets=table, where='path = %s')
        fetcher = self.one if not many else self.many
        dest[table] = fetcher(q, (relpath,))
        relations = self.schema[table].get('relations', {})
        for relation, related_tables in relations.items():
            for rel_table in related_tables:
                self._fetch(rel_table,
                            relpath,
                            dest[table],
                            many=relation == 'many')

    def get_single(self, relpath):
        q = self.db.Select(sets='content', where='path = %s')
        data = self.one(q, (relpath,))
        if data:
            for content_type, mask in metadata.CONTENT_TYPES.items():
                if data['content_type'] & mask == mask:
                    self._fetch(content_type, relpath, data)
        return data

    def get_multiple(self, relpaths, fields=None):
        q = self.db.Select(what=['*'] if fields is None else fields,
                           sets='content',
                           where=self.db.sqlin('path', relpaths))
        return self.many(q, relpaths)

    def _write(self, table_name, data, shared_data=None):
        data.update(shared_data)
        primitives = {}
        for key, value in data.items():
            if isinstance(value, dict):
                self._write(key, value, shared_data=shared_data)
            elif isinstance(value, list):
                for row in value:
                    self._write(key, row, shared_data=shared_data)
            else:
                primitives[key] = value

        constraints = self.schema[table_name]['constraints']
        q = self.db.Replace(table_name,
                            constraints=constraints,
                            cols=primitives.keys())
        self.db.execute(q, primitives)

    def add_meta_to_db(self, metadata):
        with self.db.transaction():
            replaces = metadata.get('replaces')
            self._serialize(metadata, self.transformations)
            self._write('content',
                        metadata,
                        shared_data={'path': metadata['path']})
            if replaces:
                msg = "Removing replaced content from archive database."
                logging.debug(msg)
                q = self.db.Delete('content', where='path = %s')
                self.db.execute(q, (replaces,))

        return True

    def remove_meta_from_db(self, relpath):
        with self.db.transaction():
            q = self.db.Delete('content', where='path = %s')
            rowcount = self.db.execute(q, (relpath,))
            for table in self.schema.keys():
                q = self.db.Delete(table, where='path = %s')
                self.db.execute(q, (relpath,))
            return rowcount

    def clear_and_reload(self):
        logging.debug('Content refill started.')
        q = self.db.Delete('content')
        self.db.execute(q)
        rows = self.reload_content()
        logging.info('Content refill finished for %s pieces of content', rows)

    def last_update(self):
        """ Get timestamp of the last updated content item

        :returns:  datetime object of the last updated content item
        """
        q = self.db.Select('updated',
                           sets='content',
                           order='-updated',
                           limit=1)
        res = self.db.fetchone(q)
        return res and res['updated']

    def add_view(self, relpath):
        """ Increments the viewcount for content with specified relpath

        :param relpath:  Relative path of content item
        :returns:        ``True`` if successful, ``False`` otherwise
        """
        q = self.db.Update('content', views='views + 1', where='path = %s')
        rowcount = self.db.execute(q, (relpath,))
        assert rowcount == 1, 'Updated more than one row'
        return rowcount

    def needs_formatting(self, relpath):
        """ Whether content needs formatting patch """
        q = self.db.Select('keep_formatting',
                           sets='content',
                           where='path = %s')
        result = self.db.fetchone(q, (relpath,))
        return not result['keep_formatting']

    def get_content_languages(self):
        q = 'SELECT DISTINCT language FROM content'
        languages = self.db.fetchiter(q)
        return [row['language'] for row in languages]