# Copyright (c) 2012-2013 SwiftStack, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import sys
import yaml  # *gag*; replace with msgpack!
import logging
import statlib.stats
from mako.template import Template
from collections import OrderedDict

import eventlet
import eventlet.pools
from eventlet.green.httplib import CannotSendRequest

import ssbench
import ssbench.swift_client as client
from ssbench.run_state import RunState

from pprint import pprint, pformat

def _container_creator(storage_url, token, container):
    http_conn = client.http_connection(storage_url)
    try:
        client.head_container(storage_url, token, container,
                              http_conn=http_conn)
    except client.ClientException:
        client.put_container(storage_url, token, container,
                             http_conn=http_conn)

def _gen_cleanup_job(object_info):
    return {
        'type': ssbench.DELETE_OBJECT,
        'container': object_info[0],
        'name': object_info[1],
    }

class Master:
    def __init__(self, queue, quiet=False):
        queue.watch(ssbench.STATS_TUBE)
        queue.ignore('default')
        self.queue = queue
        self.quiet = quiet

    def process_result_to(self, job, processor, label=''):
        job.delete()
        result = yaml.load(job.body)
        logging.debug('RESULT: %13s %s/%-17s %s/%s %s',
                      result['type'], result['container'], result['name'],
                      '%7.4f' % result.get('first_byte_latency')
                      if result.get('first_byte_latency', None) else ' (none)',
                      '%7.4f' % result.get('last_byte_latency')
                      if result.get('last_byte_latency', None) else '(none) ',
                      result.get('trans_id', ''))
        if label and not self.quiet:
            if 'first_byte_latency' in result:
                if result['first_byte_latency'] < 1:
                    sys.stderr.write('.')
                elif result['first_byte_latency'] < 3:
                    sys.stderr.write('o')
                elif result['first_byte_latency'] < 10:
                    sys.stderr.write('O')
                else:
                    sys.stderr.write('*')
            elif result.has_key('exception'):
                sys.stderr.write('X')
            else:
                sys.stderr.write('_')
            sys.stderr.flush()
        processor(result)

    def do_a_run(self, concurrency, job_generator, result_processor, priority,
                 storage_url, token, mapper_fn=None, label=''):
        if label and not self.quiet:
            print >>sys.stderr, label + '\n' + """
  .  <  1s first-byte-latency
  o  <  3s first-byte-latency
  O  < 10s first-byte-latency
  * >= 10s first-byte-latency
  X    work job raised an exception
  _    no first-byte-latency available
            """.strip()
        active = 0
        for job in job_generator:
            if mapper_fn is not None:
                work_job = mapper_fn(job)
                if not work_job:
                    logging.warning('Unable to fill in job %r', job)
                    continue
                job = work_job
            job['storage_url'] = storage_url
            job['token'] = token

            if active >= concurrency:
                result_job = self.queue.reserve()
                self.process_result_to(result_job, result_processor,
                                       label=label)
                active -= 1
            self.queue.put(yaml.dump(job), priority=priority)
            active += 1

            # Sink any ready results non-blockingly
            result_job = self.queue.reserve(timeout=0)
            while result_job:
                self.process_result_to(result_job, result_processor,
                                       label=label)
                active -= 1
                result_job = self.queue.reserve(timeout=0)

        # Drain the results
        while active > 0:
            result_job = self.queue.reserve()
            self.process_result_to(result_job, result_processor, label=label)
            active -= 1
        if label and not self.quiet:
            sys.stderr.write('\n')
            sys.stderr.flush()

    def run_scenario(self, auth_url, user, key, storage_url, token, scenario):
        """
        Runs a CRUD scenario, given cluster parameters and a Scenario object.

        :auth_url: Authentication URL for the Swift cluster
        :user: Account/Username to use (format is <account>:<username>)
        :key: Password for the Account/Username
        :scenario: Scenario object describing the benchmark run
        :returns: Collected result records from workers
        """

        run_state = RunState()

        self.drain_stats_queue()
        if not storage_url or not token:
            logging.debug('Authenticating to %s with %s/%s', auth_url, user,
                          key)
            storage_url, token = client.get_auth(auth_url, user, key)
        else:
            logging.debug('Using token %s at %s', token, storage_url)

        logging.info(u'Starting scenario run for "%s"', scenario.name)

        # Ensure containers exist
        logging.info('Ensuring %d containers (%s_*) exist; concurrency=%d...',
                     len(scenario.containers), scenario.container_base,
                     scenario.container_concurrency)
        pool = eventlet.GreenPool(scenario.container_concurrency)
        for container in scenario.containers:
            pool.spawn_n(_container_creator, storage_url, token, container)
        pool.waitall()

        self.queue.use(ssbench.WORK_TUBE)

        # Enqueue initialization jobs
        logging.info('Initializing cluster with stock data (up to %d '
                     'concurrent workers)', scenario.user_count)

        self.do_a_run(scenario.user_count, scenario.initial_jobs(),
                      run_state.handle_initialization_result,
                      ssbench.PRIORITY_SETUP, storage_url, token)

        logging.info('Starting benchmark run (up to %d concurrent '
                     'workers)', scenario.user_count)
        self.do_a_run(scenario.user_count, scenario.bench_jobs(),
                      run_state.handle_run_result,
                      ssbench.PRIORITY_WORK, storage_url, token,
                      mapper_fn=run_state.fill_in_job,
                      label='Benchmark Run:')

        logging.info('Deleting population objects from cluster')
        self.do_a_run(scenario.user_count,
                      run_state.cleanup_object_infos(),
                      lambda *_: None,
                      ssbench.PRIORITY_CLEANUP, storage_url, token,
                      mapper_fn=_gen_cleanup_job)

        return run_state.run_results

    def drain_stats_queue(self):
        job = self.queue.reserve(timeout=0)
        while job:
            job.delete()
            job = self.queue.reserve(timeout=0)

    def write_rps_histogram(self, stats, csv_file):
        csv_file.write('"Seconds Since Start","Requests Completed"\n')
        for i, req_count in enumerate(stats['time_series']['data'], 1):
            csv_file.write('%d,%d\n' % (i, req_count))

    def scenario_template(self):
        return """
${scenario.name}
  C   R   U   D     Worker count: ${'%3d' % agg_stats['worker_count']}   Concurrency: ${'%3d' % scenario.user_count}
%% ${'%02.0f  %02.0f  %02.0f  %02.0f' % (crud_pcts[0], crud_pcts[1], crud_pcts[2], crud_pcts[3])}
% for label, stats, sstats in stat_list:

${label}
       Count: ${'%5d' % stats['req_count']}  Average requests per second: ${'%5.1f' % stats['avg_req_per_sec']}
                            min       max      avg      std_dev    median
       First-byte latency: ${stats['first_byte_latency']['min']} - ${stats['first_byte_latency']['max']}  ${stats['first_byte_latency']['avg']}  (${stats['first_byte_latency']['std_dev']})  ${stats['first_byte_latency']['median']}  (all obj sizes)
       Last-byte  latency: ${stats['last_byte_latency']['min']} - ${stats['last_byte_latency']['max']}  ${stats['last_byte_latency']['avg']}  (${stats['last_byte_latency']['std_dev']})  ${stats['last_byte_latency']['median']}  (all obj sizes)
% for size_str, per_size_stats in sstats.iteritems():
       First-byte latency: ${per_size_stats['first_byte_latency']['min']} - ${per_size_stats['first_byte_latency']['max']}  ${per_size_stats['first_byte_latency']['avg']}  (${per_size_stats['first_byte_latency']['std_dev']})  ${per_size_stats['first_byte_latency']['median']}  (${size_str} objs)
       Last-byte  latency: ${per_size_stats['last_byte_latency']['min']} - ${per_size_stats['last_byte_latency']['max']}  ${per_size_stats['last_byte_latency']['avg']}  (${per_size_stats['last_byte_latency']['std_dev']})  ${per_size_stats['last_byte_latency']['median']}  (${size_str} objs)
% endfor
% endfor

"""

    def generate_scenario_report(self, scenario, stats):
        """Format a report based on calculated statistics for an executed
        scenario.

        :stats: A python data structure with calculated statistics
        :returns: A report (string) suitable for printing, emailing, etc.
        """

        template = Template(self.scenario_template())
        tmpl_vars = {
            'crud_pcts': scenario.crud_pcts,
            'stat_list': [
                ('TOTAL', stats['agg_stats'], stats['size_stats']),
                ('CREATE', stats['op_stats'][ssbench.CREATE_OBJECT],
                 stats['op_stats'][ssbench.CREATE_OBJECT]['size_stats']),
                ('READ', stats['op_stats'][ssbench.READ_OBJECT],
                 stats['op_stats'][ssbench.READ_OBJECT]['size_stats']),
                ('UPDATE', stats['op_stats'][ssbench.UPDATE_OBJECT],
                 stats['op_stats'][ssbench.UPDATE_OBJECT]['size_stats']),
                ('DELETE', stats['op_stats'][ssbench.DELETE_OBJECT],
                 stats['op_stats'][ssbench.DELETE_OBJECT]['size_stats']),
            ],
            'agg_stats': stats['agg_stats'],
        }
        return template.render(scenario=scenario, stats=stats, **tmpl_vars)

    def calculate_scenario_stats(self, scenario, results):
        """Given a list of worker job result dicts, compute various statistics.

        :results: A list of worker job result dicts
        :returns: A stats python dict which looks like:
            SERIES_STATS = {
                'min': 1.1,
                'max': 1.1,
                'avg': 1.1,
                'std_dev': 1.1,
                'median': 1.1,
            }
            {
                'agg_stats': {
                    'worker_count': 1,
                    'start': 1.1,
                    'stop': 1.1,
                    'req_count': 1,
                    'avg_req_per_sec': 1.1, # req_count / (stop - start)?
                    'first_byte_latency': SERIES_STATS,
                    'last_byte_latency': SERIES_STATS,
                },
                'worker_stats': {
                    1: {  # keys are worker_ids
                        'start': 1.1,
                        'stop': 1.1,
                        'req_count': 1,
                        'avg_req_per_sec': 1.1, # req_count / (stop - start)?
                        'first_byte_latency': SERIES_STATS,
                        'last_byte_latency': SERIES_STATS,
                    },
                    # ...
                },
                'op_stats': {
                    CREATE_OBJECT: { # keys are CRUD constants: CREATE_OBJECT, READ_OBJECT, etc.
                        'req_count': 1, # num requests of this CRUD type
                        'avg_req_per_sec': 1.1, # total_requests / sum(last_byte_latencies)
                        'first_byte_latency': SERIES_STATS,
                        'last_byte_latency': SERIES_STATS,
                        'size_stats': {
                            'small': { # keys are size_str values
                                'req_count': 1, # num requests of this type and size
                                'avg_req_per_sec': 1.1, # total_requests / sum(last_byte_latencies)
                                'first_byte_latency': SERIES_STATS,
                                'last_byte_latency': SERIES_STATS,
                            },
                            # ...
                        },
                    },
                    # ...
                },
                'size_stats': {
                    'small': { # keys are size_str values
                        'req_count': 1, # num requests of this size (for all CRUD types)
                        'avg_req_per_sec': 1.1, # total_requests / sum(last_byte_latencies)
                        'first_byte_latency': SERIES_STATS,
                        'last_byte_latency': SERIES_STATS,
                    },
                    # ...
                },
                'time_series': {
                    'start': 1, # epoch time of first data point
                    'data': [
                        1, # number of requests finishing during this second
                        # ...
                    ],
                },
            }
        """
        # Each result looks like:
        # {
        #   'worker_id': 1,
        #   'type': 'get_object',
        #   'size': 4900000,
        #   'first_byte_latency': 0.9137639999389648,
        #   'last_byte_latency': 0.913769006729126,
        #   'completed_at': 1324372892.360802,
        #}
        #OR
        # {
        #   'worker_id': 1,
        #   'type': 'get_object',
        #   'completed_at': 1324372892.360802,
        #   'exception': '...',
        # }
        logging.info('Calculating statistics for %d result items...',
                     len(results))
        agg_stats = dict(start=2**32, stop=0, req_count=0)
        op_stats = {}
        for crud_type in [ssbench.CREATE_OBJECT, ssbench.READ_OBJECT,
                          ssbench.UPDATE_OBJECT, ssbench.DELETE_OBJECT]:
            op_stats[crud_type] = dict(
                req_count=0, avg_req_per_sec=0,
                size_stats=OrderedDict.fromkeys(scenario.sizes_by_name.keys()))

        req_completion_seconds = {}
        completion_time_max = 0
        completion_time_min = 2**32
        stats = dict(
            agg_stats=agg_stats,
            worker_stats={},
            op_stats=op_stats,
            size_stats=OrderedDict.fromkeys(scenario.sizes_by_name.keys()))
        for result in results:
            if 'exception' in result:
                # skip but log exceptions
                logging.warn('calculate_scenario_stats: exception from '
                             'worker %d: %s',
                             result['worker_id'], result['exception'])
                continue
            completion_time = int(result['completed_at'])
            if completion_time < completion_time_min:
                completion_time_min = completion_time
            if completion_time > completion_time_max:
                completion_time_max = completion_time
            req_completion_seconds[completion_time] = \
                1 + req_completion_seconds.get(completion_time, 0)
            result['start'] = (
                result['completed_at'] - result['last_byte_latency'])

            # Stats per-worker
            if result['worker_id'] not in stats['worker_stats']:
                stats['worker_stats'][result['worker_id']] = {}
            self._add_result_to(stats['worker_stats'][result['worker_id']],
                                result)

            # Stats per-file-size
            if not stats['size_stats'][result['size_str']]:
                stats['size_stats'][result['size_str']] = {}
            self._add_result_to(stats['size_stats'][result['size_str']],
                                result)

            self._add_result_to(agg_stats, result)
            self._add_result_to(op_stats[result['type']], result)

            # Stats per-operation-per-file-size
            if not op_stats[result['type']]['size_stats'][result['size_str']]:
                op_stats[result['type']]['size_stats'][result['size_str']] = {}
            self._add_result_to(
                op_stats[result['type']]['size_stats'][result['size_str']],
                result)
        agg_stats['worker_count'] = len(stats['worker_stats'].keys())
        self._compute_req_per_sec(agg_stats)
        self._compute_latency_stats(agg_stats)
        for worker_stats in stats['worker_stats'].values():
            self._compute_req_per_sec(worker_stats)
            self._compute_latency_stats(worker_stats)
        for op_stat, op_stats_dict in op_stats.iteritems():
            if op_stats_dict['req_count']:
                self._compute_req_per_sec(op_stats_dict)
                self._compute_latency_stats(op_stats_dict)
                for size_str, size_stats in \
                        op_stats_dict['size_stats'].iteritems():
                    if size_stats:
                        self._compute_req_per_sec(size_stats)
                        self._compute_latency_stats(size_stats)
                    else:
                        op_stats_dict['size_stats'].pop(size_str)
        for size_str, size_stats in stats['size_stats'].iteritems():
            if size_stats:
                self._compute_req_per_sec(size_stats)
                self._compute_latency_stats(size_stats)
            else:
                stats['size_stats'].pop(size_str)
        time_series_data = [req_completion_seconds.get(t, 0)
                            for t in range(completion_time_min,
                                           completion_time_max + 1)]
        stats['time_series'] = dict(start=completion_time_min,
                                    data=time_series_data)

        return stats

    def _compute_latency_stats(self, stat_dict):
        try:
            for latency_type in ('first_byte_latency', 'last_byte_latency'):
                stat_dict[latency_type] = self._series_stats(
                    stat_dict[latency_type])
        except KeyError:
            logging.exception('stat_dict: %r', stat_dict)
            raise

    def _compute_req_per_sec(self, stat_dict):
        stat_dict['avg_req_per_sec'] = round(stat_dict['req_count'] /
                                             (stat_dict['stop'] -
                                              stat_dict['start']), 6)

    def _add_result_to(self, stat_dict, result):
        if 'start' not in stat_dict or result['start'] < stat_dict['start']:
            stat_dict['start'] = result['start']
        if 'stop' not in stat_dict or \
                result['completed_at'] > stat_dict['stop']:
            stat_dict['stop'] = result['completed_at']
        if 'req_count' not in stat_dict:
            stat_dict['req_count'] = 1
        else:
            stat_dict['req_count'] += 1
        self._rec_latency(stat_dict, result)

    def _series_stats(self, sequence):
        pre_filter = len(sequence)
        sequence = filter(None, sequence)
        logging.debug('_series_stats pre/post seq len: %d/%d', pre_filter,
                      len(sequence))
        if not sequence:
            # No data available
            return dict(min=' N/A  ', max='  N/A  ', avg='  N/A  ',
                        std_dev='  N/A  ', median='  N/A  ')
        try:
            n, (minval, maxval), mean, std_dev, skew, kurtosis = \
                statlib.stats.ldescribe(sequence)
        except ZeroDivisionError:
            # Handle the case of a single-element sequence (population standard
            # deviation divides by N-1)
            minval = sequence[0]
            maxval = sequence[0]
            mean = sequence[0]
            std_dev = 0
        return dict(
            min='%6.3f' % minval,
            max='%7.3f' % maxval,
            avg='%7.3f' % mean,
            std_dev='%7.3f' % statlib.stats.lsamplestdev(sequence),
            median='%7.3f' % statlib.stats.lmedianscore(sequence))

    def _rec_latency(self, stats_dict, result):
        for latency_type in ('first_byte_latency', 'last_byte_latency'):
            if latency_type in stats_dict:
                stats_dict[latency_type].append(result[latency_type])
            else:
                stats_dict[latency_type] = [result[latency_type]]


import random

class InputsConfig(object):
    #################################################################### simulation parameters that needs to be set up #####################################################################

    ##### Blocks parameters #####
    Binterval = 12.42 #596 # Average time (in seconds)for creating a block in the blockchain
    Bsize = 7997148 #875225 the block limit in Gas/ byte
    Bdelay = 2.3#0.42 # average block propogation delay in seconds, #Ref: https://bitslog.wordpress.com/2016/04/28/uncle-mining-an-ethereum-consensus-protocol-flaw/
    Breward = 3 # Reward for mining a block

    ##### Transactions parameters #####
    hasTrans = False  # True/False to enable/disable transactions in the simulator
    Ttechnique = "Full" # Full/Light to specify the way of modelling transactions
    Type="ETH" # style ETH or BTC
    Tn=5 # rate of the number of transactions created per second
    Tdelay = 5.1 # average transaction propagation delay in seconds (Only if Full technique is used)
    Tfee = 0.000000021 # average transaction fee per size unit
    Tsize = 546 # average transaction size  in byte
    ## note: the transaction fee (the submitter of the transaction has to pay) is Tfee * Tsize

    ##### Uncles parameters #####
    hasUncles = True # boolean variable to indicate use of uncle mechansim or not
    Buncles=2 # maximum number of uncle blocks allowed per block
    Ugenerations = 7 # the depth in which an uncle can be included in a block
    Ureward =0
    UIreward = Breward / 32 # Reward for including an uncle

    ##### Nodes parameters #####
    Nn = 3 # the total number of nodes in the network
    # the id & hash power for each node in the network -> id should be strat from 0
    # hashPower =  random.uniform(0,30)
    # NODE = Node(id, hashPower)
    # NODES = []
    # NODES += NODE
    NODES_INFO =[] #IP and Port info of all other nodes

    total_hashpower = 0
    # for i in range(Nn):
    #     hashPower =  random.uniform(0,30) # generate a random hash power for node i between 0% and 30%
    #     NODES += [Node(i,hashPower)]
    # IP = gethostbyname(socket. gethostname())
    # PORT = 8001

    ######### if you want to configure the hash power of each node as you prefer use this instead of randomly assign hash powers to nodes ############
    # NODES = [
    # Node(0,15.2),
    #     Node(1,13.8),
    #     Node(2,13.5),
    #     Node(3,12.6),
    #     Node(4,11.3),
    #     Node(5,10.30),
    #     Node(6,9.2),
    #     Node(7,6.1),
    #     Node(8,2.5),
    #     Node(9,1.3),
    #     Node(10,1),
    #     Node(11,0.8),
    #     Node(12,0.8),
    #     Node(13,0.8),
    #     Node(14,0.6),
    #     Node(15,2),
    # ]

    ##### Simulation parameters ####
    simTime= 1000 # the simulation length (in seconds)
    Runs=1# Number of simulation runs

    ########################################################### The distribution for transactions' attributes in the Ethereum network (gathered from real Ethereum)  ###########################################################################################
    def random_Gaslimit():
        global glimit
        rand = random.uniform(0,1)
        if rand <= 0.15:
            glimit = 21000
        elif rand <= 0.32:
            glimit = random.randint(21001,50000)
        elif rand <= 0.71:
            glimit = random.randint(50001,100000)
        elif rand <= 0.93:
            glimit = random.randint(100001,250000)
        elif rand <= 0.98:
            glimit = random.randint(250001,1000000)
        elif rand <= 1:
            glimit = random.randint(1000001,7919992)
        return glimit

    def random_Usedgas():
        global ugas
        rand = random.uniform(0,1)
        if rand <= 0.14:
            ugas = random.uniform(0.0,0.25)
        elif rand <= 0.38:
            ugas = random.uniform(0.26,0.50)
        elif rand <= 0.65:
            ugas = random.uniform(0.51,0.75)
        elif rand <= 1:
            ugas = random.uniform(0.76,1)
        return ugas

    def random_GasPrice():
        global gprice
        rand = random.uniform(0,1)
        if rand <= 0.13:
            gprice = random.uniform(0.000000000,0.000000004)
        elif rand <= 0.46:
            gprice = random.uniform(0.000000005,0.00000001)
        elif rand <= 0.72:
            gprice = random.uniform(0.000000011,0.00000002)
        elif rand <= 0.89:
            gprice = random.uniform(0.000000021,0.00000004)
        elif rand <= 1:
            gprice = random.uniform(0.000000041,0.000000134)
        return gprice


# 计算测试数据

import numpy as np
import utils

if __name__ == '__main__':
    malls = utils.get_malls()
    conn = utils.get_db_conn()
    cur = conn.cursor()
    i = 1
    for mall_id in malls:
        print(utils.get_time(), ' ','start handle mall ', mall_id)
        # 获取模型
        model = utils.get_model_xgb(mall_id)
        if model == 0:
            print('no model for mall ', mall_id)
            continue
        # 查出所有wifi，排序
        sql = 'SELECT DISTINCT wifi_ssid FROM {m} ORDER BY wifi_ssid'.format(m=mall_id)
        cur.execute(sql)
        wifis = [r[0] for r in cur.fetchall()]
        # 初始化数据矩阵和初始向量
        metrix = []
        vec = [0 for wifi in range(0, len(wifis))]
        rows = []
        # 查询所有数据
        sql = "SELECT row_id,wifi_ssid,wifi_db FROM data_test_final WHERE mall_id='%s' ORDER BY data_id,wifi_ssid " % mall_id
        cur.execute(sql)
        row = cur.fetchone()
        v = vec[:]
        row_id = row[0]
        if wifis.__contains__(row[1]):
            v[wifis.index(row[1])] = utils.normal(row[2])
        for r in cur.fetchall():
            # 根据是否与前一条row_id相同进行不同操作
            if r[0] != row_id:
                metrix.append(v)
                rows.append(row_id)
                v = vec[:]
                row_id = r[0]
            if wifis.__contains__(r[1]):
                v[wifis.index(r[1])] = utils.normal(r[2])
        metrix.append(v)
        rows.append(row_id)
        metrix = np.array(metrix)
        result = model.predict(metrix)
        # print(result)
        # print(cur.rowcount)
        # print(len(result))
        # print(len(rows))
        for r in range(0, len(rows)):
            sql = "INSERT INTO data_test_result VALUES ('{r}','{s}')".format(r = rows[r], s=result[r])
            cur.execute(sql)
        sql = "INSERT INTO data_test_handled SET mall_id='%s'" % mall_id
        cur.execute(sql)
        conn.commit()
        print(utils.get_time(), ' ',mall_id, ' handled done')
        print(i, ' handled.')
        i += 1


import time
import playsound
import os
import random
from gtts.lang import tts_langs
from deep_translator import (GoogleTranslator)
import pickle
import json

# my classes
import random_test
import mike
import lang_tests
import level_one
import level_two
import level_three
import extras

# game progress vars
parts = {"l1_colours": "0", "l1_numbers": "0", "l1_animals": "0", "l2_greetings": "0", "l2_transport": "0",
         "l2_sports": "0", "l3_food": "0", "l3_clothes": "0", "l3_buildings": "0"}
progress = {"name": "", "language": "",
            "level": "0", "lvlpts": "0", "partsDone": parts, "points": "0", "saveTime": "0"}

def set_l1(p1=False, p2=False, p3=False, all=False):
    if all:
        parts['l1_colours'] = '1'
        parts['l1_animals'] = '1'
        parts['l1_numbers'] = '1'
    else:
        if p1:
            parts['l1_colours'] = p1
        if p2:
            parts['l1_animals'] = p2
        if p3:
            parts['l1_numbers'] = p3

def set_l2(p1=False, p2=False, p3=False, all=False):
    if all:
        parts['l2_greetings'] = '1'
        parts['l2_transport'] = '1'
        parts['l2_sports'] = '1'
    else:
        if p1:
            parts['l2_greetings'] = p1
        if p2:
            parts['l2_transport'] = p2
        if p3:
            parts['l2_sports'] = p3

def set_l3(p1=False, p2=False, p3=False, all=False):
    if all:
        parts['l3_food'] = '1'
        parts['l3_clothes'] = '1'
        parts['l3_buildings'] = '1'
    else:
        if p1:
            parts['l3_food'] = p1
        if p2:
            parts['l3_clothes'] = p2
        if p3:
            parts['l3_buildings'] = p3

# LANG METHODS

def get_name():
    confirmed = False
    mike.mike('Welcome to uno lingo. What is your name?')
    name = mike.record_audio()
    print(name)
    while confirmed == False:
        voice_confirm = mike.record_audio(
            "Your name is " + name + ". Is that correct? Say yes to confirm")
        if "yes" in voice_confirm:
            confirmed = True
        else:
            name = mike.record_audio('What is your name?')
            print(name)

    mike.mike('Nice to meet you, ' + name+'.')
    mike.mike("My name is Mike.")
    return name

def choose_language():
    valid = False
    chosen = False
    langs_dict = GoogleTranslator.get_supported_languages(as_dict=True)
    print("Languages available: ")
    for key in langs_dict:
        if langs_dict[key] in tts_langs():
            print(key)

    while valid == False:
        while chosen == False:
            choice = mike.record_audio("Please select a new language to learn")
            try:
                languageToLearn = extras.get_language_short(choice)
                if languageToLearn in tts_langs():
                    valid = True
                if valid == True:
                    response = mike.record_audio(
                        "You have chosen " + choice+", say yes to start learning this language.")
                    if 'yes' in response:
                        chosen = True
                        return languageToLearn
            except:
                return 0

# SETUP - LEVELS
def get_level():
    response = ''
    name = progress['name']
    mike.mike(
        name + ", would you like to take a test to find out what language level you are on?")
    while len(response) < 1:
        response = mike.record_audio()
        if 'yes' in response:
            level = test_Level()
        elif 'no' in response:
            mike.mike('What level are you on?')
            level = mike.record_audio()
            level = int(level)
            while level > 3 or level < 1:
                level = mike.record_audio(
                    "I'm sorry, please choose a level between one and four.")
    set_parts(level)
    progress['partsDone'] = parts
    return str(level)

def test_Level():
    percent = 0
    percent = lang_tests.test_l1(progress['language'])
    if percent >= 70:
        percent = lang_tests.test_l2(progress['language'])
        if percent >= 70:
            level = 3
        else:
            level = 2
    elif percent >= 40:
        level = 2
    else:
        level = 1

    mike.mike("The tests have determined you are at level " + str(level))
    return level

# PLAY - LEVELS

def begin_level():
    # "switch" for levels here
    if progress['level'] == '1':
        load_lvl_1()
    elif progress['level'] == '2':
        load_lvl_2()
    elif progress['level'] == '3':
        load_lvl_3()

    return 0

def set_parts(lvl, p1=False, p2=False, p3=False):

    if lvl == "1":
        set_l1(p1, p2, p3)
    elif lvl == "2":
        set_l1(all="1")
        set_l2(p1, p2, p3)
    elif lvl == "3":
        set_l1(all="1")
        set_l2(all="1")
        set_l3(p1, p2, p3)

def part_choice(lvl):

    user_input = "   "
    top = []
    topicStr = ""
    validChoice = False
    if lvl == '1':
        top.append(progress['partsDone']['l1_colours'])
        top.append(progress['partsDone']['l1_animals'])
        top.append(progress['partsDone']['l1_numbers'])

        topics = ["colours", "animals", "numbers"]
    elif lvl == '2':
        top.append(progress['partsDone']['l2_greetings'])
        top.append(progress['partsDone']['l2_transport'])
        top.append(progress['partsDone']['l2_sports'])

        topics = ["greetings", "transport", "sports"]
    elif lvl == '3':
        print("level: ", lvl)
        top.append(progress['partsDone']['l3_food'])
        top.append(progress['partsDone']['l3_clothes'])
        top.append(progress['partsDone']['l3_buildings'])

        topics = ["food", "clothes", "buildings"]

    else:
        Exception
    topic = []
    j = 0
    while j < 3:
        if top[j] == "0":
            topic.append(str(topics[j]))
            topicStr = topicStr + str(topics[j]) + ", "
        j = j+1

    topicStr = topicStr[:-2]

    if topicStr == "":
        return 10
    else:
        user_input = mike.record_audio(
            "Please select a topic. Your options include " + topicStr)

        while validChoice == False:
            print("You said: " + user_input)

            if user_input == "":
                user_input = "    "
            if user_input in topicStr:
                validChoice = True
                # return user_input
            else:
                if user_input == "    ":
                    user_input = mike.record_audio(
                        "That is not a valid option. Please select one of the following: " + topicStr)
                else:
                    user_input = mike.record_audio(
                        user_input + " is not a valid option. Please select one of the following: " + topicStr)
        x = 0
        while x < len(topics):
            if user_input in str(topics[x]):
                part_no = x
                return part_no
            x = x+1
        return part_no

# Easiest level, teach colours, numbers and animals

def load_lvl_1():
    save_progress()

    mike.mike("\n\nLevel One")
    print("---------------------------------------------------------")
    load_this = part_choice('1')
    if load_this == 0:
        l1_colours()
        load_lvl_1()
    if load_this == 1:
        l1_animals()
        load_lvl_1()
    if load_this == 2:
        l1_numbers()
        load_lvl_1()
    if load_this == 10:
        mike.mike("\n\nCongratulations, you have finished level one!")
        print("Total points this level: ", progress['points'])

        load_lvl_2()

def l1_numbers():
    points = progress['points']
    lan = progress['language']
    newP = level_one.numbers(lan)
    print("---------------------------------------------------------")
    print("Points this level: ", newP,
          "\nPossible points this level: 11")
    points = int(points)+newP
    print("Total points: ", points)
    points = str(points)
    # set this part to done so it won't be available again
    progress['partsDone']['l1_numbers'] = "1"
    progress['points'] = points

def l1_animals():
    points = progress['points']

    lan = progress['language']

    newP = level_one.animals(lan)
    print("---------------------------------------------------------")
    print("Points this level: ", newP,
          "\nPossible points this level: 11")
    points = int(points)+newP

    print("Total points: ", points)
    points = str(points)

    # set this part to done so it won't be available again
    progress['partsDone']['l1_animals'] = "1"
    progress['points'] = points

def l1_colours():
    points = progress['points']

    lan = progress['language']
    newP = level_one.colours(lan)
    print("---------------------------------------------------------")
    print("Points this level: ", newP,
          "\nPossible points this level: 11")
    points = int(points)+newP
    print("Total points: ", points)
    points = str(points)

    # set this part to done so it won't be available again
    progress['partsDone']['l1_colours'] = "1"
    progress['points'] = points

# Slightly more difficult level, teach more complex words and some adjectives

def load_lvl_2():
    save_progress()
    mike.mike("\n\nLevel Two")
    print("---------------------------------------------------------")
    load_this = part_choice('2')
    if load_this == 0:
        l2_greetings()
        load_lvl_2()
    if load_this == 1:
        l2_transport()
        load_lvl_2()
    if load_this == 2:
        l2_sports()
        load_lvl_2()
    if load_this == 10:
        mike.mike("\n\nCongratulations, you have finished level two!")
        print("Total points this level: ", progress['points'])
        load_lvl_3()

# Medium level, teach basic sentences

def l2_transport():
    points = progress['points']

    lan = progress['language']
    newP = level_two.transport(lan)
    print("---------------------------------------------------------")
    print("Points this level: ", newP,
          "\nPossible points this level: 11")
    points = int(points)+newP

    print("Total points: ", points)
    points = str(points)

    # set this part to done so it won't be available again
    progress['partsDone']['l2_transport'] = "1"
    progress['points'] = points

def l2_sports():
    points = progress['points']

    lan = progress['language']
    newP = level_two.sports(lan)
    print("---------------------------------------------------------")
    print("Points this level: ", newP,
          "\nPossible points this level: 11")
    points = int(points)+newP

    print("Total points: ", points)
    points = str(points)

    # set this part to done so it won't be available again
    progress['partsDone']['l2_sports'] = "1"
    progress['points'] = points

def l2_greetings():
    points = progress['points']

    lan = progress['language']
    newP = level_two.greetings(lan)
    print("---------------------------------------------------------")
    print("Points this level: ", newP,
          "\nPossible points this level: 11")
    points = int(points)+newP

    print("Total points: ", points)
    points = str(points)

    # set this part to done so it won't be available again
    progress['partsDone']['l2_greetings'] = "1"
    progress['points'] = points

def load_lvl_3():

    save_progress()

    mike.mike("\n\nLevel Three")
    print("---------------------------------------------------------")
    load_this = part_choice('3')
    if load_this == 0:
        l3_food()
        load_lvl_3()
    if load_this == 1:
        l3_clothes()
        load_lvl_3()
    if load_this == 2:
        l3_buildings()
        load_lvl_3()
    if load_this == 10:
        mike.mike("\n\nCongratulations, you have finished level three!")
        print("Total points this level: ", progress['points'])

def l3_food():
    points = progress['points']

    lan = progress['language']
    newP = level_three.food(lan)
    print("---------------------------------------------------------")
    print("Points this level: ", newP,
          "\nPossible points this level: 11")
    points = int(points)+newP

    print("Total points: ", points)
    points = str(points)

    # set this part to done so it won't be available again
    progress['partsDone']['l3_food'] = "1"
    progress['points'] = points

def l3_clothes():
    points = progress['points']

    lan = progress['language']
    newP = level_three.clothes(lan)
    print("---------------------------------------------------------")
    print("Points this level: ", newP,
          "\nPossible points this level: 11")
    points = int(points)+newP

    print("Total points: ", points)
    points = str(points)

    # set this part to done so it won't be available again
    progress['partsDone']['l3_clothes'] = "1"
    progress['points'] = points

def l3_buildings():
    points = progress['points']

    lan = progress['language']
    newP = level_three.buildings(lan)
    print("---------------------------------------------------------")
    print("Points this level: ", newP,
          "\nPossible points this level: 11")
    points = int(points)+newP

    print("Total points: ", points)
    points = str(points)

    # set this part to done so it won't be available again
    progress['partsDone']['l3_buildings'] = "1"
    progress['points'] = points

def graduate():
    mike.mike("Congratulations, you have completed the course")
    print("---------------------------------------------------------")

    user_answer = mike.record_audio("Say new game to learn a new language.")
    return user_answer

# SAVES

def save_progress():
    res = load_progress(progress['name'])
    newPrint = res

    check = "'language': " + "'"+str(progress['language']+"'")
    progress['saveTime'] = extras.make_time()

    if(not(check in str(res))):
        try:
            with open('saves/'+progress['name'] + '.txt', 'ab+') as f:
                pickle.dump(progress, f)

        except:
            os.makedirs("saves")
            with open('saves/'+progress['name'] + '.txt', 'ab+') as f:
                pickle.dump(progress, f)
    else:
        with open('saves/'+progress['name'] + '.txt', 'wb') as f:
            pickle.dump("", f)

        x = 0
        while x < len(newPrint):
            if check in str(newPrint[x]):
                if str(progress['saveTime'] >= newPrint[x]['saveTime']):
                    if progress in newPrint:
                        del newPrint[x]
                    else:
                        newPrint[x] = progress

            x = x + 1
        for n in newPrint:
            with open('saves/'+progress['name'] + '.txt', 'ab+') as f:
                pickle.dump(n, f)

def load_progress(n):
    results = []
    try:
        with open('saves/'+n + '.txt', 'rb') as f:
            try:
                while True:
                    results.append(pickle.load(f))
            except EOFError:
                pass
        return results
    except:
        print("no saved file")
        return 0

def check_saves(name, language=False):
    langsSaved = []
    savedP = load_progress(name)
    if language:
        try:
            savedP = str(savedP).replace('\'', '\"')
            ps = json.loads(str(savedP))
            for p in ps:
                if p['language'] == language:
                    level = p['level']
                    progress['partsDone'] = p['partsDone']
            return level
        except:
            return 0
    else:
        try:
            savedP = str(savedP).replace("'',", "")
            savedP = str(savedP).replace('\'', '\"')
            ps = json.loads(str(savedP))
            for p in ps:
                langsSaved.append(p['language'])
            return langsSaved
        except:
            return langsSaved

def choose_save(saves):
    valid = False

    mike.mike("saved languages include ")
    for s in saves:
        mike.mike(extras.get_language_long(s))
    language = mike.record_audio(
        "Which language would you like to continue learning?")
    print("You have chosen: " + language)
    while valid == False:
        try:
            language = extras.get_language_short(language)
        except:
            return

        if language in saves:
            level = check_saves(progress['name'], language)
            progress['language'] = language
            progress['level'] = str(level)
            valid = True
        else:
            language = mike.record_audio(
                "I'm sorry, that is not a valid option. Choose a different language.")
            print("You have chosen: " + language)

# PART METHODS

def startup():
    # get users name
    name = get_name()
    progress['name'] = name

    # check if user has saved file
    saves = check_saves(name)
    if saves:
        answered = False
        answer = ""
        while answered == False:
            answer = mike.record_audio(
                "You have saved progress available. Would you like to continue progress?")
            if "yes" in answer:
                answered = True
                progBool = True
            if "no" in answer:
                answered = True
                progBool = False

    else:
        progBool = False
    if progBool == True:
        # if yes - ask which save to load
        choose_save(saves)
        return
    else:
        # ask language to learn
        progress['language'] = choose_language()
        progress['level'] = get_level()
        save_progress()
        return

# TEST METHODS

def print_prog():
    print("Name: " + progress['name'])
    print("Language: " + progress['language'])
    print("Level: " + progress['level'])
    print("Parts: ", progress['partsDone'])

def easy_set(name=False, lang=False, level=False):
    if name and lang and level:
        progress['name'] = name
        progress['language'] = lang
        progress['level'] = level
    else:
        progress['name'] = "John"
        progress['language'] = 'es'
        progress['level'] = '1'

    set_parts(progress['level'])

if __name__ == "__main__":
    answer = "new game"
    while(answer == "new game"):
        startup()
        begin_level()
        answer = graduate()
        if answer == "quit game":
            mike.mike("Goodbye, ", progress['name'])
            exit()


"""Request context."""

import json
import re
from collections import OrderedDict
from enum import Enum
from typing import Optional, Tuple, MutableMapping, Any, Union

from aiohttp import web

from .const import JSONAPI
from .errors import HTTPBadRequest
from .log import logger
from .schema import Schema
from .schema.common import Event

FILTER_KEY = re.compile(r"filter\[(?P<field>\w[-\w_]*)\]")
FILTER_VALUE = re.compile(r"(?P<name>[a-z]+):(?P<rule>.*)")
FIELDS_RE = re.compile(r"fields\[(?P<name>\w[-\w_]*)\]")

class SortDirection(Enum):
    ASC = '+'
    DESC = '-'

class RequestContext:
    def __init__(self, request: web.Request, resource_type: str = None):
        self._pagination = None
        self._resource_type = resource_type
        self.request = request
        self.filters = self.parse_request_filters(request)
        self.fields = self.parse_request_fields(request)
        self.include = self.parse_request_includes(request)
        self.sorting = self.parse_request_sorting(request)
        if self.request.method in Event.__members__:
            self.event = Event[self.request.method]
        else:
            self.event = None

    @property
    def schema(self) -> Optional[Schema]:
        registry = self.request.app[JSONAPI]['registry']
        return registry.get(self._resource_type, None)

    @property
    def pagination(self):
        if self._pagination is not None:
            return self._pagination

        if self.schema is not None:
            pagination_type = self.schema.opts.get('pagination')
            if pagination_type:
                self._pagination = pagination_type(self.request)
                return self._pagination

        return None

    @staticmethod
    def parse_request_filters(
        request: web.Request) -> MutableMapping[Tuple[str, str], Any]:
        """
        .. hint::

            Please note, that the *filter* strategy is not defined by the
            JSON API specification and depends on the implementation.
            If you want to use another filter strategy,
            feel free to **override** this method.

        Returns a OrderedDict with tuples (field, name) as keys
        and rule as values. Rule value is JSON deserialized from query string.

        Filters can be applied using the query string.

        .. code-block:: python3

            >>> from aiohttp_json_api.context import RequestContext
            >>> from aiohttp.test_utils import make_mocked_request

            >>> request = make_mocked_request('GET', '/api/User/?filter[name]=endswith:"Simpson"')
            >>> RequestContext.parse_request_filters(request)
            OrderedDict([(('name', 'endswith'), 'Simpson')])

            >>> request = make_mocked_request('GET', '/api/User/?filter[name]=endswith:"Simpson"&filter[name]=in:["Some","Names"]')
            >>> RequestContext.parse_request_filters(request)
            OrderedDict([(('name', 'endswith'), 'Simpson'), (('name', 'in'), ['Some', 'Names'])])

            >>> request = make_mocked_request('GET', '/api/User/?filter[name]=in:["Homer Simpson", "Darth Vader"]')
            >>> RequestContext.parse_request_filters(request)
            OrderedDict([(('name', 'in'), ['Homer Simpson', 'Darth Vader'])])

            >>> request = make_mocked_request('GET', '/api/User/?filter[email]=startswith:"lisa"&filter[age]=lt:20')
            >>> RequestContext.parse_request_filters(request)
            OrderedDict([(('email', 'startswith'), 'lisa'), (('age', 'lt'), 20)])

        The general syntax is::

            "?filter[field]=name:rule"

        where *rule* is a JSON value.

        :raises HTTPBadRequest:
            If the rule of a filter is not a JSON object.
        :raises HTTPBadRequest:
            If a filtername contains other characters than *[a-z]*.
        """
        filters = OrderedDict()
        for key, value in request.query.items():
            key_match = re.fullmatch(FILTER_KEY, key)
            value_match = re.fullmatch(FILTER_VALUE, value)

            # If the key indicates a filter, but the value is not correct
            # formatted.
            if key_match and not value_match:
                field = key_match.group('field')
                raise HTTPBadRequest(
                    detail="The filter '{}' "
                           "is not correct applied.".format(field),
                    source_parameter=key
                )

            # The key indicates a filter and the filter name exists.
            elif key_match and value_match:
                field = key_match.group('field')
                name = value_match.group('name')
                rule = value_match.group('rule')
                try:
                    rule = json.loads(rule)
                except Exception as err:
                    logger.debug(err, exc_info=False)
                    raise HTTPBadRequest(
                        detail="The rule '{}' "
                               "is not JSON serializable".format(rule),
                        source_parameter=key
                    )
                filters[(field, name)] = rule
        return filters

    @staticmethod
    def parse_request_fields(
        request: web.Request) -> MutableMapping[str, Tuple[str, ...]]:
        """
        The fields, which should be included in the response (sparse fieldset).

        .. code-block:: python3

            >>> from aiohttp_json_api.context import RequestContext
            >>> from aiohttp.test_utils import make_mocked_request
            >>> request = make_mocked_request('GET', '/api/User?fields[User]=email,name&fields[Post]=comments')
            >>> RequestContext.parse_request_fields(request)
            OrderedDict([('User', ('email', 'name')), ('Post', ('comments',))])

        :seealso: http://jsonapi.org/format/#fetching-sparse-fieldsets
        """
        fields = OrderedDict()
        for key, value in request.query.items():
            match = re.fullmatch(FIELDS_RE, key)
            if match:
                typename = match.group('name')
                type_fields = tuple(item.strip()
                                    for item in value.split(',')
                                    if item.strip())

                fields[typename] = type_fields
        return fields

    @staticmethod
    def parse_request_includes(request: web.Request) -> Tuple[Tuple[str], ...]:
        """
        Returns the names of the relationships, which should be included into
        the response.

        .. code-block:: python3

            >>> from aiohttp_json_api.context import RequestContext
            >>> from aiohttp.test_utils import make_mocked_request
            >>> request = make_mocked_request('GET', '/api/Post?include=author,comments.author')
            >>> RequestContext.parse_request_includes(request)
            (('author',), ('comments', 'author'))

        :seealso: http://jsonapi.org/format/#fetching-includes
        """
        include = request.query.get('include', '')
        return tuple(
            tuple(path.split('.')) for path in include.split(',') if path
        )

    @staticmethod
    def parse_request_sorting(
        request: web.Request) -> MutableMapping[Tuple[str, ...], SortDirection]:
        """
        Returns a mapping with tuples as keys, and values with SortDirection,
        describing how the output should be sorted.

        .. code-block:: python3

            >>> from aiohttp_json_api.context import RequestContext
            >>> from aiohttp.test_utils import make_mocked_request
            >>> request = make_mocked_request('GET', '/api/Post?sort=name,-age,+comments.count')
            >>> RequestContext.parse_request_sorting(request)
            OrderedDict([(('name',), <SortDirection.ASC: '+'>), (('age',), <SortDirection.DESC: '-'>), (('comments', 'count'), <SortDirection.ASC: '+'>)])

        :seealso: http://jsonapi.org/format/#fetching-sorting
        """
        sort_in_query = request.query.get('sort')
        sort_in_query = sort_in_query.split(',') if sort_in_query else list()

        sort = OrderedDict()

        for field in sort_in_query:
            if field[0] == '+' or field[0] == '-':
                direction = SortDirection(field[0])
                field = field[1:]
            else:
                direction = SortDirection.ASC

            field = tuple(e.strip() for e in field.split('.'))
            sort[field] = direction

        return sort

    def has_filter(self, field: str, name: str) -> bool:
        """
        Returns true, if the filter *name* has been applied at least once
        on the *field*.

        :arg str field:
            Name of field
        :arg str name:
            Name of filter
        """
        return (field, name) in self.filters

    def get_filter(self, field: str, name: str, default: Any = None) -> Any:
        """
        If the filter *name* has been applied on the *field*, the
        *filter* is returned and *default* otherwise.

        :arg str field:
            Name of field
        :arg str name:
            Name of filter
        :arg Any default:
            A fallback rule value for the filter.
        """
        return self.filters.get((field, name), default)

    def get_order(self, field: Union[str, Tuple[str, ...]],
                  default: SortDirection = SortDirection.ASC) -> SortDirection:
        """
        Checks if a sort criterion (``+`` or ``-``) for the *field* exists
        and returns it.

        :arg Union[str, Tuple[str, ...]] field:
        :arg SortDirection default:
            Returned, if no criterion is set by the request.
        """
        field = tuple(field.split('.')) if isinstance(field, str) else field
        return self.sorting.get(field, default)


# O(N)
import unittest

def urlify_algo(string, length):
    """replace spaces with %20 and removes trailing spaces"""
    # convert to list because Python strings are immutable
    char_list = list(string)
    string = ""
    new_index = len(char_list)

    for i in reversed(range(length)):
        if char_list[i] == " ":
            # Replace spaces
            char_list[new_index - 3 : new_index] = "%20"
            new_index -= 3
        else:
            # Move characters
            char_list[new_index - 1] = char_list[i]
            new_index -= 1
    # convert back to string
    return string.join(char_list)

def urlify_pythonic(text, length):
    """solution using standard library"""
    return text.rstrip().replace(" ", "%20")

class Test(unittest.TestCase):
    """Test Cases"""

    test_cases = [
        ("much ado about nothing      ", "much%20ado%20about%20nothing"),
        ("Mr John Smith    ", "Mr%20John%20Smith"),
    ]
    testable_functions = [urlify_algo, urlify_pythonic]

    def test_urlify(self):
        for urlify in self.testable_functions:
            for test_string, expected in self.test_cases:
                stripped_length = len(test_string.rstrip(" "))
                actual = urlify(test_string, stripped_length)
                self.assertEqual(actual, expected)

if __name__ == "__main__":
    unittest.main()


import os, itertools, time, pickle
import subprocess
from xml.dom import minidom
from collections import Counter, OrderedDict
from operator import itemgetter
from scipy import spatial
from sklearn.metrics import precision_score, accuracy_score, recall_score, f1_score
from sklearn.feature_extraction.text import TfidfVectorizer
import re, sys
import numpy as np
import scipy.sparse as sp
import torch
from torch import nn
from torch import optim
import torch.nn.functional as F
from math import ceil, exp
from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence

f = open("data.pkl", "rb")
data, emb_indexer, emb_indexer_inv, emb_vals, gt_mappings  = pickle.load(f)

ontologies_in_alignment = [l.split(".")[0].split("-") for l in os.listdir("reference-alignment/")]
flatten = lambda l: [item for sublist in l for item in sublist]

ind_test, inp_test1, inp_test2 = None, None, None

def write(statement):
    op_file = open("Logs", "a+")
    op_file.write("\n" + str(statement) + "\n")
    op_file.close()

class Ontology():
    def __init__(self, ontology):
        self.ontology = ontology
        self.ontology_obj = minidom.parse(ontology)
        self.root = self.ontology_obj.documentElement
        self.subclasses = self.parse_subclasses()
        self.object_properties = self.parse_object_properties()
        self.data_properties = self.parse_data_properties()
        self.triples = self.parse_triples()
        self.classes = self.parse_classes()

    def get_child_node(self, element, tag):
        return [e for e in element._get_childNodes() if type(e)==minidom.Element and e._get_tagName() == tag]

    def has_attribute_value(self, element, attribute, value):
        return True if element.getAttribute(attribute).split("#")[-1] == value else False

    def get_subclass_triples(self):
        return [(b,a,"subclass_of") for (a,b) in self.get_subclasses()]

    def parse_triples(self, union_flag=0, subclass_of=True):
        obj_props = self.object_properties
        data_props = self.data_properties
        props = obj_props + data_props
        all_triples = []
        for prop in props:
            domain_children = self.get_child_node(prop, "rdfs:domain")
            range_children = self.get_child_node(prop, "rdfs:range")
            domain_prop = self.filter_null([self.extract_ID(el) for el in domain_children])
            range_prop = self.filter_null([self.extract_ID(el) for el in range_children])
            if not domain_children or not range_children:
                continue
            if not domain_prop:
                domain_prop = self.filter_null([self.extract_ID(el) for el in domain_children[0].getElementsByTagName("owl:Class")])
            if not range_prop:
                range_prop = self.filter_null([self.extract_ID(el) for el in range_children[0].getElementsByTagName("owl:Class")])
            if domain_prop and range_prop:
                if union_flag == 0:
                    all_triples.extend([(el[0], el[1], self.extract_ID(prop)) for el in list(itertools.product(domain_prop, range_prop))])
                else:
                    all_triples.append(("###".join(domain_prop), "###".join(range_prop), self.extract_ID(prop)))
        if subclass_of:
            all_triples.extend(self.get_subclass_triples())
        return list(set(all_triples))

    def get_triples(self, union_flag=0, subclass_of=True, include_inv=True):
        return self.parse_triples(union_flag, subclass_of)

    def parse_subclasses(self, union_flag=0):
        subclasses = self.root.getElementsByTagName("rdfs:subClassOf")
        subclass_pairs = []
        for el in subclasses:
            inline_subclasses = self.extract_ID(el)
            if inline_subclasses:
                subclass_pairs.append((el, el.parentNode))
            else:
                level1_class = self.get_child_node(el, "owl:Class")
                if not level1_class:
                    continue
                if self.extract_ID(level1_class[0]):
                    subclass_pairs.append((level1_class[0], el.parentNode))
                else:
                    level2classes = level1_class[0].getElementsByTagName("owl:Class")

                    subclass_pairs.extend([(elem, el.parentNode) for elem in level2classes if self.extract_ID(elem)])
        return subclass_pairs

    def get_subclasses(self):
        return [(self.extract_ID(a), self.extract_ID(b)) for (a,b) in self.subclasses]

    def filter_null(self, data):
        return [el for el in data if el]

    def extract_ID(self, element):
        element_id = element.getAttribute("rdf:ID") or element.getAttribute("rdf:resource") or element.getAttribute("rdf:about")
        return element_id.split("#")[-1]

    def parse_classes(self):
        class_elems = [self.extract_ID(el) for el in self.root.getElementsByTagName("owl:Class")]
        subclass_classes = list(set(flatten([el[:-1] for el in self.triples])))
        return list(set(self.filter_null(class_elems + subclass_classes)))

    def get_classes(self):
        return self.classes

    def get_entities(self):
        entities = [self.extract_ID(el) for el in self.root.getElementsByTagName("owl:Class")]
        return list(set(self.filter_null(entities)))

    def parse_data_properties(self):
        data_properties = [el for el in self.get_child_node(self.root, 'owl:DatatypeProperty')]
        fn_data_properties = [el for el in self.get_child_node(self.root, 'owl:FunctionalProperty') if el]
        fn_data_properties = [el for el in fn_data_properties if type(el)==minidom.Element and
            [el for el in self.get_child_node(el, "rdf:type") if
             self.has_attribute_value(el, "rdf:resource", "DatatypeProperty")]]
        inv_fn_data_properties = [el for el in self.get_child_node(self.root, 'owl:InverseFunctionalProperty') if el]
        inv_fn_data_properties = [el for el in inv_fn_data_properties if type(el)==minidom.Element and
            [el for el in self.get_child_node(el, "rdf:type") if
             self.has_attribute_value(el, "rdf:resource", "DatatypeProperty")]]
        return data_properties + fn_data_properties + inv_fn_data_properties

    def parse_object_properties(self):
        obj_properties = [el for el in self.get_child_node(self.root, 'owl:ObjectProperty')]
        fn_obj_properties = [el for el in self.get_child_node(self.root, 'owl:FunctionalProperty') if el]
        fn_obj_properties = [el for el in fn_obj_properties if type(el)==minidom.Element and
            [el for el in self.get_child_node(el, "rdf:type") if
             self.has_attribute_value(el, "rdf:resource", "ObjectProperty")]]
        inv_fn_obj_properties = [el for el in self.get_child_node(self.root, 'owl:InverseFunctionalProperty') if el]
        inv_fn_obj_properties = [el for el in inv_fn_obj_properties if type(el)==minidom.Element and
            [el for el in self.get_child_node(el, "rdf:type") if
             self.has_attribute_value(el, "rdf:resource", "ObjectProperty")]]
        return obj_properties + fn_obj_properties + inv_fn_obj_properties

    def get_object_properties(self):
        obj_props = [self.extract_ID(el) for el in self.object_properties]
        return list(set(self.filter_null(obj_props)))

    def get_data_properties(self):
        data_props = [self.extract_ID(el) for el in self.data_properties]
        return list(set(self.filter_null(data_props)))

def greedy_matching():
    global batch_size, test_data_t, test_data_f, model, optimizer, emb_indexer_inv, gt_mappings, all_metrics
    all_results = OrderedDict()
    with torch.no_grad():
        all_pred = []
        batch_size = min(batch_size, len(test_data_t))
        num_batches = int(ceil(len(test_data_t)/batch_size))
        batch_size_f = int(ceil(len(test_data_f)/num_batches))

        np.random.shuffle(test_data_t)
        np.random.shuffle(test_data_f)

        for batch_idx in range(num_batches):
            batch_start = batch_idx * batch_size
            batch_end = (batch_idx+1) * batch_size

            batch_start_f = batch_idx * batch_size_f
            batch_end_f = (batch_idx+1) * batch_size_f

            pos_elems = np.array(test_data_t)[batch_start:batch_end]
            neg_elems = np.array(test_data_f)[batch_start_f:batch_end_f]
            optimizer.zero_grad()

            inputs = np.array([generate_data(elem) for elem in list(pos_elems) + list(neg_elems)])
            targets = np.array([1 for i in range(len(pos_elems))] + [0 for i in range(len(neg_elems))])

            indices = np.random.permutation(inputs.shape[0])
            inputs, targets = inputs[indices].transpose(1,0,2), targets[indices]
            inputs_elem = inputs.copy()

            nonzero_elems = np.count_nonzero(inputs, axis=-1)
            indices = np.flip(np.argsort(nonzero_elems, axis=-1), axis=-1)
            seq_lens = np.flip(np.sort(nonzero_elems, axis=-1), axis=-1)
            inputs = np.stack((inputs[0][[indices[0]]], inputs[1][[indices[1]]]), axis=0)

            d1 = {elem:i for i,elem in enumerate(indices[0])}
            d2 = {elem:i for i,elem in enumerate(indices[1])}
            rev_indices = np.stack(([d1[k] for k in range(inputs_elem.shape[1])], [d2[k] for k in range(inputs_elem.shape[1])]))

            rev_indices = torch.LongTensor(rev_indices.T).to(device)
            inputs = torch.LongTensor(inputs.transpose(1,0,2)).to(device)
            seq_lens = torch.LongTensor(seq_lens.copy().T).to(device)
            targets = torch.DoubleTensor(targets)

            outputs = model(inputs, seq_lens, rev_indices)
            outputs = [el.item() for el in outputs]
            #outputs /= torch.sum(outputs, dim=1).view(-1, 1)
            #outputs = [(1-el[1].item()) for el in outputs]

            targets = [True if el.item() else False for el in targets]
#             print (inputs)
            for idx, pred_elem in enumerate(outputs):
                ent1 = emb_indexer_inv[inputs_elem[0][idx][0]]
                ent2 = emb_indexer_inv[inputs_elem[1][idx][0]]
                if (ent1, ent2) in all_results:
                    print ("Error: ", ent1, ent2, "already present")
                all_results[(ent1, ent2)] = (pred_elem, targets[idx])

        #all_results = OrderedDict(sorted(all_results.items(), key=lambda x: x[0], reverse=True))
        #filtered_results = dict()

        #entities_to_assign = set([el[0] for el in list(all_results.keys())])
        #for pair in all_results:
        #    if pair[0] in entities_to_assign:
        #        filtered_results[pair] = all_results[pair]
        #        entities_to_assign.remove(pair[0])

        #entities_to_assign = set([el[1] for el in list(all_results.keys())])
        #for pair in all_results:
        #    if pair[1] in entities_to_assign:
        #        filtered_results[pair] = all_results[pair]
        #        entities_to_assign.remove(pair[1])

        #filtered_results = OrderedDict(sorted(filtered_results.items(), key=lambda x: x[1][0], reverse=True))

        optimum_metrics, opt_threshold = [-1000 for i in range(5)], -1000
        low_threshold = np.min([el[0] for el in all_results.values()]) - 0.02
        high_threshold = np.max([el[0] for el in all_results.values()]) + 0.02
        #low_threshold, high_threshold = 0.9, 1.02
        threshold = low_threshold
        step = 0.01
        while threshold < high_threshold:
            res = []
            for i,key in enumerate(all_results):
                if all_results[key][0] > threshold:
                    res.append(key)
            fn_list = [key for key in gt_mappings if key not in set(res) and not is_valid(test_onto, key)]
            fp_list = [elem for elem in res if not all_results[elem][1]]
            tp_list = [elem for elem in res if all_results[elem][1]]

            tp, fn, fp = len(tp_list), len(fn_list), len(fp_list)
            exception = False

            try:
                precision = tp/(tp+fp)
                recall = tp/(tp+fn)
                f1score = 2 * precision * recall / (precision + recall)
                f2score = 5 * precision * recall / (4 * precision + recall)
                f0_5score = 1.25 * precision * recall / (0.25 * precision + recall)
            except Exception as e:
                print (e)
                exception = True
                step = 0.01
                threshold += step
                continue
            print ("Threshold: ", threshold, precision, recall, f1score, f2score, f0_5score)

            if f1score > optimum_metrics[2]:
                optimum_metrics = [precision, recall, f1score, f2score, f0_5score]
                opt_threshold = threshold

            if threshold > 0.98 and not exception:
                step = 0.0001
            else:
                step = 0.01
            print (step, threshold, exception)
            threshold += step
        print ("Precision: {} Recall: {} F1-Score: {} F2-Score: {} F0.5-Score: {}".format(*optimum_metrics))
        if optimum_metrics[2] != -1000:
            all_metrics.append((opt_threshold, optimum_metrics))
    return all_results

def write(elem):
    f = open("Logs", "a+")
    if type(elem) == list or type(elem) == tuple:
        string = str("\n".join([str(s) for s in elem]))
    else:
        string = str(elem)
    f.write("\n"+string)
    f.close()

inputs3, results3 = None, None

class SiameseNetwork(nn.Module):
    def __init__(self, embedding_dim, hidden_dim, num_layers):
        super().__init__()

        self.embedding_dim = embedding_dim
        self.hidden_dim = hidden_dim
        self.num_layers = num_layers
        self.num_directions = 2

        self.name_embedding = nn.Embedding(len(emb_vals), self.embedding_dim)
        self.name_embedding.load_state_dict({'weight': torch.from_numpy(np.array(emb_vals))})
        self.name_embedding.weight.requires_grad = False

        self.dropout = dropout

        self.lstm = nn.LSTM(self.embedding_dim, self.hidden_dim, self.num_layers, bidirectional=True, batch_first=True)
        self.cosine_sim_layer = nn.CosineSimilarity(dim=1)
        self.bilinear = nn.Bilinear(self.hidden_dim, self.hidden_dim, 1)

    def forward(self, inputs, seq_lens, rev_indices):
        results = []
        inputs = inputs.permute(1,0,2)
        seq_lens, rev_indices = seq_lens.T, rev_indices.T
        #print ("input len: {} seq len: {}, rev len: {}".format(inputs.shape, seq_lens.shape, rev_indices.shape))
        for i in range(2):
            x = self.name_embedding(inputs[i])
            #self.lstm.flatten_parameters()
            packed_inp = pack_padded_sequence(x, seq_lens[i].cpu().numpy(), batch_first=True)
            op, (ht, ct) = self.lstm(x)
            x = ht[2*(self.num_layers-1):].permute(1,0,2)
            x = x[rev_indices[i],:,:]
            results.append(x.reshape(-1, self.num_directions*self.hidden_dim))
        #global inputs3, results3
        #results3 = results
        #inputs3 = inputs
        #x = self.layer1(results[0], results[1])
        #x = F.log_softmax(x)
        x = self.cosine_sim_layer(results[0], results[1])
        return x

def get_one_hop_neighbours(ont, K=1):
    ont_obj = Ontology("conference_ontologies/" + ont + ".owl")
    triples = ont_obj.get_triples()
    entities = [(a,b) for (a,b,c) in triples]
    neighbours_dict = {elem: [elem] for elem in list(set(flatten(entities)))}
    for e1, e2 in entities:
        neighbours_dict[e1].append(e2)
        neighbours_dict[e2].append(e1)

    prop_triples = ont_obj.get_triples(subclass_of=False)
    neighbours_dict_props = {c: [c] for a,b,c in prop_triples}
    for e1, e2, p in prop_triples:
        neighbours_dict_props[p].extend([e1, e2])

    neighbours_dict = {**neighbours_dict, **neighbours_dict_props}

    for elem in ont_obj.get_entities() + ont_obj.get_object_properties() + ont_obj.get_data_properties():
        if elem not in neighbours_dict:
            neighbours_dict[elem] = [elem]

    neighbours_dict = {el: neighbours_dict[el][:1] + sorted(list(set(neighbours_dict[el][1:])))
                       for el in neighbours_dict}
    neighbours_dict = {el: neighbours_dict[el][:int(sys.argv[1])] for el in neighbours_dict}
    neighbours_dict = {ont + "#" + el: [ont + "#" + e for e in neighbours_dict[el]] for el in neighbours_dict}
    return neighbours_dict

def is_valid(test_onto, key):
    return tuple([el.split("#")[0] for el in key]) not in test_onto

def generate_data(elem_tuple):
    op = np.array([[emb_indexer[el] for el in neighbours_dicts[elem.split("#")[0]][elem]] for elem in elem_tuple])
    return op

def generate_input(elems, target):
    inputs = np.array([generate_data(elem) for elem in list(elems)])
    targets = np.array([target for i in range(len(elems))])
    return inputs, targets

neighbours_dicts = {ont: get_one_hop_neighbours(ont) for ont in list(set(flatten(ontologies_in_alignment)))}
max_neighbours = np.max(flatten([[len(el[e]) for e in el] for el in neighbours_dicts.values()]))
neighbours_lens = {ont: {key: len(neighbours_dicts[ont][key]) for key in neighbours_dicts[ont]}
                   for ont in neighbours_dicts}
neighbours_dicts = {ont: {key: neighbours_dicts[ont][key] + ["<UNK>" for i in range(max_neighbours -len(neighbours_dicts[ont][key]))]
              for key in neighbours_dicts[ont]} for ont in neighbours_dicts}

print("Number of neighbours: " + sys.argv[1])

data_items = data.items()
np.random.shuffle(list(data_items))
data = OrderedDict(data_items)

print ("Number of entities:", len(data))
all_ont_pairs = list(set([tuple([el.split("#")[0] for el in l]) for l in data.keys()]))

all_metrics = []

for i in list(range(0, len(all_ont_pairs), 3)):

    test_onto = all_ont_pairs[i:i+3]

    train_data = {elem: data[elem] for elem in data if tuple([el.split("#")[0] for el in elem]) not in test_onto}
    test_data = {elem: data[elem] for elem in data if tuple([el.split("#")[0] for el in elem]) in test_onto}

    torch.set_default_dtype(torch.float64)

    train_test_split = 0.9

    train_data_t = [key for key in train_data if data[key]]
    train_data_f = [key for key in train_data if not data[key]]
    #train_data_f = train_data_f[:int(len(train_data_t))]
#     [:int(0.1*(len(train_data) - len(train_data_t)) )]
#     np.random.shuffle(train_data_f)

    lr = 0.001
    num_epochs = 50
    weight_decay = 0.001
    batch_size = 8
    dropout = 0.3
    batch_size = min(batch_size, len(train_data_t))
    num_batches = int(ceil(len(train_data_t)/batch_size))
    batch_size_f = int(ceil(len(train_data_f)/num_batches))
    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

    model = SiameseNetwork(512, 250, 1).to(device)

    optimizer = optim.Adam(model.parameters(), lr=lr, weight_decay=weight_decay)

    for epoch in range(num_epochs):
        inputs_pos, targets_pos = generate_input(train_data_t, 1)
        inputs_neg, targets_neg = generate_input(train_data_f, 0)
        indices_pos = np.random.permutation(len(inputs_pos))
        indices_neg = np.random.permutation(len(inputs_neg))

        inputs_pos, targets_pos = inputs_pos[indices_pos], targets_pos[indices_pos]
        inputs_neg, targets_neg = inputs_neg[indices_neg], targets_neg[indices_neg]

#        indices = np.random.permutation(len(inputs_pos) + len(inputs_neg))

#        inputs = np.array(list(inputs_pos) + list(inputs_neg))[indices]
#        targets = np.array(list(targets_pos) + list(targets_neg))[indices]

#         inputs = np.array(list(inputs_pos) + list(inputs_neg))
#         targets = np.array(list(targets_pos) + list(targets_neg))

        for batch_idx in range(num_batches):
            batch_start = batch_idx * batch_size
            batch_end = (batch_idx+1) * batch_size

            batch_start_f = batch_idx * batch_size_f
            batch_end_f = (batch_idx+1) * batch_size_f

            inputs = np.concatenate((inputs_pos[batch_start: batch_end], inputs_neg[batch_start_f: batch_end_f]))
            targets = np.concatenate((targets_pos[batch_start: batch_end], targets_neg[batch_start_f: batch_end_f]))

            inp = inputs.transpose(1,0,2)
            nonzero_elems = np.count_nonzero(inp, axis=-1)
            indices = np.flip(np.argsort(nonzero_elems, axis=-1), axis=-1)
            seq_lens = np.flip(np.sort(nonzero_elems, axis=-1), axis=-1)
            inp_elems = np.stack((inp[0][[indices[0]]], inp[1][[indices[1]]]), axis=0).transpose(1,0,2)

            inp_elems = torch.LongTensor(inp_elems).to(device)
            targ_elems = torch.DoubleTensor(targets).to(device)
            optimizer.zero_grad()

            d1 = {elem:i for i,elem in enumerate(indices[0])}
            d2 = {elem:i for i,elem in enumerate(indices[1])}
            rev_indices = np.stack(([d1[k] for k in range(inp.shape[1])],
                                    [d2[k] for k in range(inp.shape[1])]))

            rev_indices = torch.LongTensor(rev_indices.T).to(device)
            seq_lens = torch.LongTensor(seq_lens.copy().T).to(device)
            outputs = model(inp_elems, seq_lens, rev_indices)
            loss = F.mse_loss(outputs, targ_elems)
            loss.backward()
#             break
            optimizer.step()

            if batch_idx%10 == 0:
                print ("Epoch: {} Idx: {} Loss: {}".format(epoch, batch_idx, loss.item()))

    model.eval()

    test_data_t = [key for key in test_data if data[key]]
    test_data_f = [key for key in test_data if not data[key]]

    res = greedy_matching()

print ("Final Results: " + str(np.mean([el[1] for el in all_metrics], axis=0)))
print ("Best threshold: " + str(all_metrics[np.argmax([el[1][2] for el in all_metrics])][0]))


import torch
import torch.nn as nn
import torch.utils.model_zoo as model_zoo
import numpy as np
import time

from receptive_field import compute_proto_layer_rf_info

img_size = 224

model_urls = {
    'vgg11': 'https://download.pytorch.org/models/vgg11-bbd30ac9.pth',
    'vgg13': 'https://download.pytorch.org/models/vgg13-c768596a.pth',
    'vgg16': 'https://download.pytorch.org/models/vgg16-397923af.pth',
    'vgg19': 'https://download.pytorch.org/models/vgg19-dcbb9e9d.pth',
    'vgg11_bn': 'https://download.pytorch.org/models/vgg11_bn-6002323d.pth',
    'vgg13_bn': 'https://download.pytorch.org/models/vgg13_bn-abd245e5.pth',
    'vgg16_bn': 'https://download.pytorch.org/models/vgg16_bn-6c64b313.pth',
    'vgg19_bn': 'https://download.pytorch.org/models/vgg19_bn-c79401a0.pth',
}

class VGG_proto(nn.Module):

    def __init__(self, features, root, proto_layer_rf_info, num_prototypes_per_class, prototype_dimension, init_weights=True):
        super(VGG_proto, self).__init__()

        self.root = root
        self.num_prototypes_per_class = num_prototypes_per_class
        self.prototype_dimension = prototype_dimension

        for name, num_children in root.class_to_num_children().items():
            setattr(self,"num_" + name, num_children)

        for name,shape in root.class_to_proto_shape(x_per_child=num_prototypes_per_class, dimension=prototype_dimension).items():
            setattr(self,name + "_prototype_shape",shape)
            setattr(self,"num_" + name + "_prototypes",shape[0])
            setattr(self,name + "_prototype_vectors", nn.Parameter(torch.rand(shape), requires_grad=True))
            setattr(self,name + "_layer", nn.Linear(shape[0], getattr(self,"num_" + name), bias = False))
            setattr(self,"ones_" + name, nn.Parameter(torch.ones(shape), requires_grad=False))

            root.set_node_attr(name,"num_prototypes_per_class",num_prototypes_per_class)
            root.set_node_attr(name,"prototype_shape",shape)

        self.epsilon = 1e-4

        self.proto_layer_rf_info = proto_layer_rf_info

        self.features = features # this has to be named features to allow the precise loading
        self.add_on_layers = nn.Sequential(
            nn.Conv2d(in_channels=512, out_channels=prototype_dimension, kernel_size=1),
            nn.ReLU(),
            nn.Conv2d(in_channels=prototype_dimension, out_channels=prototype_dimension, kernel_size=1),
            nn.Sigmoid()
        )

        if init_weights:
            self._initialize_weights()

    def forward(self, x):

        conv_features = self.conv_features(x)

        for node in self.root.nodes_with_children():
            self.classifier(conv_features,node)

        return "remember logits and distances attached to nodes"

    def classifier(self, conv_features, node):
        distances = self.prototype_distances(conv_features, node.name)
        min_distances = -nn.functional.max_pool2d(-distances,
                                                  kernel_size=(distances.size()[2], distances.size()[3])) # global max pooling
        min_distances = min_distances.view(-1, getattr(self,"num_" + node.name + "_prototypes"))
        prototype_activations = torch.log(1 + (1 / (min_distances + self.epsilon)))
        logits = getattr(self, node.name + "_layer")(prototype_activations)
        setattr(node,"logits",logits)
        setattr(node,"min_distances",min_distances)
        return None

    def _l2_convolution(self, x, name):
        '''
        apply self.prototype_vectors as l2-convolution filters on input x
        '''

        x2 = x ** 2
        x2_patch_sum = nn.functional.conv2d(input=x2, weight= getattr(self,"ones_" + name))
        p2 = getattr(self,name + "_prototype_vectors") ** 2
        p2 = torch.sum(p2, dim=(1, 2, 3)) # p2 is a vector of shape (num_prototypes,)
        xp = nn.functional.conv2d(input=x, weight= getattr(self,name + "_prototype_vectors"))
        p2_reshape = p2.view(-1, 1, 1)
        intermediate_result = - 2 * xp + p2_reshape # use broadcast
        distances = nn.functional.relu(x2_patch_sum + intermediate_result) # x2_patch_sum and intermediate_result are of the same shape

        return distances

    def conv_features(self, x):
        x = self.features(x)
        x = self.add_on_layers(x)
        return x

    def prototype_distances(self, conv_features, name):
        '''
        x is the raw input
        '''
        x = self._l2_convolution(conv_features, name)
        return x

    def remaining_layers(self, x):
        x = -nn.functional.max_pool2d(-x, kernel_size=(x.size()[2], x.size()[3])) # global max pooling
        x = x.view(-1, self.num_prototypes)
        x = - torch.log(x + self.epsilon)
        x = self.last_layer(x)
        return x

    def _initialize_weights(self):
        for m in self.modules(): # Returns an iterator over all modules in the network.
            if len(m._modules) > 0: # skip anything that's not a single layer
                continue
            if isinstance(m, nn.Conv2d):
                # every init technique has an underscore _ in the name
                # nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
                nn.init.normal_(m.weight, mean=0, std=0.1)
                if m.bias is not None:
                    nn.init.constant_(m.bias, 0)
            elif isinstance(m, nn.BatchNorm2d):
                nn.init.constant_(m.weight, 1)
                nn.init.constant_(m.bias, 0)
            elif isinstance(m, nn.Linear):
                identity = torch.eye(m.out_features)
                repeated_identity = identity.unsqueeze(2).repeat(1,1,self.num_prototypes_per_class).\
                                            view(m.out_features, -1)
                m.weight.data.copy_(1.5 * repeated_identity - 0.5)

    def _smart_init_fine_protos(self):
        # init the fine protos around the coarse protos belonging to their respective classes
        # for each coarse class, iterate through its coarse protos dropping one new fine proto near it each time

        sd = .03

        for node in self.root.nodes_with_children():
            if node.name != "root":
                prototype_vectors = getattr(self,node.name + "_prototype_vectors")
                parent_prototype_vectors = getattr(self,node.parent.name + "_prototype_vectors")
                for i in range(prototype_vectors.size(0)):
                    j = i % self.num_prototypes_per_class
                    mean = torch.rand((self.prototype_dimension,1,1))
                    mean.data.copy_(parent_prototype_vectors[j,:,:,:].data)
                    prototype_vectors[i,:,:,:].data.copy_(nn.init.normal_(mean,sd))

    def get_joint_distribution(self):

        batch_size = self.root.logits.size(0)

        #top_level = torch.nn.functional.softmax(self.root.logits,1)
        top_level = self.root.logits
        bottom_level = self.root.distribution_over_furthest_descendents(batch_size)

        names = self.root.unwrap_names_of_joint(self.root.names_of_joint_distribution())
        idx = np.argsort(names)

        bottom_level = bottom_level[:,idx]

        return top_level, bottom_level

def make_layers(cfg, batch_norm=False):
    # cfg configuration
    layers = []
    in_channels = 3
    for v in cfg:
        if v == 'M':
            layers += [nn.MaxPool2d(kernel_size=2, stride=2)]
        else:
            conv2d = nn.Conv2d(in_channels, v, kernel_size=3, padding=1)
            if batch_norm:
                layers += [conv2d, nn.BatchNorm2d(v), nn.ReLU(inplace=True)]
            else:
                layers += [conv2d, nn.ReLU(inplace=True)]
            in_channels = v
    return nn.Sequential(*layers) # unravel the list to arguments of Sequential

cfg = {
    'A': [64, 'M', 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'],
    'B': [64, 64, 'M', 128, 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'],
    'D': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, 512, 512, 'M'],
    '256': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 256, 256, 256, 'M', 256, 256, 256, 'M'],
    '128': [64, 64, 'M', 128, 128, 'M', 128, 128, 128, 'M', 128, 128, 128, 'M', 128, 128, 128, 'M'],
    'E': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 256, 'M', 512, 512, 512, 512, 'M', 512, 512, 512, 512, 'M'],
}

def vgg16_proto(root, pretrained=False, num_prototypes_per_class = 1, prototype_dimension = 512, model_path = None, resume_path = None, **kwargs):
    """VGG 16-layer model (configuration "D")
    Args:
        pretrained (bool): If True, returns a model pre-trained on ImageNet
    """
    cfg_ = cfg['D']
    model = VGG_proto(features=make_layers(cfg_), root=root, num_prototypes_per_class = num_prototypes_per_class, prototype_dimension = prototype_dimension,
                      proto_layer_rf_info=compute_proto_layer_rf_info(img_size, cfg_, 1), init_weights=True)

    if pretrained:

        if resume_path is not None:
            print("loading model from %s" % resume_path)
            model.load_state_dict(torch.load(resume_path))

        else: # expects path to VGG model
            print("loading model from %s" % model_path)
            state_dict = torch.load(model_path)

            new_state_dict = dict()
            for k, v in state_dict.items():
                name = k
                new_state_dict[name] = v

            keys = [key for key in new_state_dict.keys()]
            for key in keys:
                if key.startswith('classifier'): # the classifier weights are the fully connected layers in vgg which we have removed
                    del new_state_dict[key]

            model.load_state_dict(new_state_dict, strict = False)

    return model


# !/usr/bin/env python
# -*- coding: utf-8 -*-

''' a sheduled WebPageCollector Python 3
'''

from Agenda import Agenda, DailyAgenda, WeeklyAgenda
from URLRoller import URLRoller
from Spiders import ThesesSpider
from CfgReader import CfgFileParser
from FileLogger import FileLogger
import pymongo
from pymongo import  MongoClient
from pymongo.errors  import ConnectionFailure
from twisted.internet import reactor, defer
import scrapy
from scrapy.crawler import CrawlerRunner,Crawler

import datetime
import time
import json

CONFIGFILE= 'WebPageCollector.cfg'

#============================================================================================================
class WebPageCollector():

    def __init__(self):
        cfg_parser = CfgFileParser()
        self.Config = cfg_parser.file_reader(CONFIGFILE)
        self.Project = self.Config['Collector']['project']
        self.Drop = self.Config[self.Project]['drop'] == 'True'
        self.Cycle = self.Config[self.Project]['cycle'] == 'True'
        self.Next = self.Config[self.Project]['next'] == 'True'
        self.mongo_db = self.Config[self.Project]['mongo_db']
        self.Host = self.Config[self.Project]['host']
        self.Port = int(self.Config[self.Project]['port'])
        self.Collection = self.Config[self.Project]['collection']
        self.mongo_client = MongoClient(self.Host, self.Port)
        self.DBase = self.mongo_client[self.mongo_db]
        if self.Drop :
            self.DBase[self.Collection].drop()
        self.file_logger = FileLogger()
        self.set_up()

    def set_up(self):
        PipeLines = self.Config[self.Project]['pipelines']
        PipeLines = json.loads(PipeLines)
        self.spider_settings = {
                            'USER_AGENT': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36 OPR/38.0.2220.41',
                            'DOWNLOADER_MIDDLEWARES ': {'scrapy.downloadermiddlewares.cookies.CookiesMiddleware': 543},
                            'DOWNLOAD_DELAY': 3,
                            'RANDOMIZE_DOWNLOAD_DELAY': True,
                            'COOKIES_DEBUG': False,
                            'COOKIES_ENABLED': True,
                            'ITEM_PIPELINES': PipeLines,
                            'LOG_LEVEL' : 'INFO'
                         }
        self.SpiderName = self.Config[self.Project]['spider']
        self.URLRoller = URLRoller()
        if self.Config['Agenda']['pace'] == 'daily':
            self.Agenda = DailyAgenda()
        if self.Config['Agenda']['pace'] == 'weekly':
            self.Agenda = WeeklyAgenda()
        self.file_logger.csv_log('COLLECTOR', 'SetUp is done')

    def Crawl_job(self,URL,Next):
        Runner = CrawlerRunner(settings=self.spider_settings)
        return Runner.crawl(eval(self.SpiderName),profile_url=URL,next=Next)

    def Schedule_next_crawl(self,null,Now):
        self.file_logger.csv_log('COLLECTOR', 'Crawl Session will start')
        if self.Agenda.Begin <= Now and Now < self.Agenda.End :
            delay = self.Agenda.Delay(Now)
            self.file_logger.csv_log('COLLECTOR', 'delayed by',delay)
            reactor.callLater(delay,self.LoopCapturing)
        else:
            self.file_logger.csv_log('COLLECTOR', 'Crawl Session is ended')
            reactor.stop()

    def Capturing(self):
        Now = datetime.datetime.now()
        URL = self.URLRoller.next()
        print(Now.time(),'->',URL)
        Job = self.Crawl_job(URL,self.Next)
        Job.addCallback(self.Schedule_next_crawl,Now)
        Job.addErrback(self.errback)

    def CaptureOnes(self,URL):
        print('->',URL)
        Now = datetime.datetime.now()
        Job = self.Crawl_job(URL,False)
        Job.addCallback(self.Schedule_next_crawl,Now)
        Job.addErrback(self.errback)

    def LoopCapturing(self):
        Now = datetime.datetime.now()
        URL = self.URLRoller.next()
        if URL == self.URLRoller.starter() and not self.Cycle :
            reactor.stop()
        else:
            print(Now.time(),'->',URL)
            Job = self.Crawl_job(URL,self.Next)
            Job.addCallback(self.Schedule_next_crawl,Now)
            Job.addErrback(self.errback)

    def errback(self,failure):
        print('ERROR:',failure.value)
        self.file_logger.csv_log('COLLECTOR', failure.value, self.start_urls[0])

#============================================================================================================
if __name__ == '__main__':
    WPC = WebPageCollector()
    WPC.Capturing()
    reactor.run()


import re

from django.db.models import Count, Q
from django.shortcuts import render, redirect
from django_filters.rest_framework import DjangoFilterBackend
from rest_framework import status
from rest_framework.decorators import action
from rest_framework.filters import OrderingFilter
from rest_framework.response import Response
from rest_framework.viewsets import ModelViewSet
from webweb import Web

from microservices.filters import MicroserviceFilter
from microservices.models import Microservice, Author, Tag, Company, Pair
from microservices.serializers import MicroserviceSerializer, AuthorSerializer, TagSerializer, CompanySerializer, \
    MicroserviceReadSerializer, PairSerializer, AuthorReadSerializer, CompanyReadSerializer

webweb_file = 'webweb_representation.html'

def draw_graph_and_save_to_file():
    pairs = Pair.objects.all().distinct()
    adjacency = [[pair.first_microservice.name, pair.second_microservice.name] for pair in pairs]

    display = {
        'attachWebwebToElementWithId': None,
        'charge': 2500,
        'colorBy': 'degree',
        'colorPalette': 'Set1',
        'height': 500,
        'linkLength': 200,
        'linkStrength': 0.75,
        'radius': 10,
        'scaleLinkWidth': True,
        'showNodeNames': True,
        'sizeBy': 'weightedDegree',
        'width': 1400,
    }

    with open('tmp.txt', 'w', encoding='utf-8') as f:
        f.write(str(adjacency))
    # draw_graph_and_save_to_file()
    web = Web(adjacency=adjacency, display=display, title="All microservices")
    return web.html, adjacency

def index(request, microservices=None):
    html_code, adjacency = draw_graph_and_save_to_file()
    html_code = re.sub(
        '"networks": {}',
        f'"networks": {{"All microservices": {{"layers": [{{"edgeList": {adjacency}, "nodes": {{}}, "metadata": null}}]}}}}',
        html_code
    )
    html_code = re.sub('#webweb-center {(.*?)}', r'#webweb-center {\1 \n margin-top: 1150px; display:inline-block;}',
                       html_code, flags=re.DOTALL)
    styles = '\n'.join(re.findall('<style>(.*?)</style>', html_code, re.DOTALL))
    # styles += """#webweb-menu-right {display: grid;}\n#webweb-menu-left {display: grid;}"""
    scripts = '\n'.join(re.findall('<script.*?</script>', html_code, re.DOTALL))
    # context = {'html_code': html_code}
    if microservices is None:
        microservices = MicroserviceReadSerializer(Microservice.objects.all().order_by('-modified')[:10],
                                                   many=True).data
    context = {
        'styles': styles,
        'scripts': scripts,
        'authors': AuthorReadSerializer(Author.objects.all(), many=True).data,
        'statuses': [{'key': status[0], 'display_name': status[1]} for status in Microservice.STATUS],
        'companies': CompanyReadSerializer(Company.objects.all(), many=True).data,
        'microservices': microservices
    }
    return render(request, 'microservices/index.html', context)

def add_microservice(request):
    context = {
        'authors': AuthorReadSerializer(Author.objects.all(), many=True).data,
        'statuses': [{'key': status[0], 'display_name': status[1]} for status in Microservice.STATUS],
        'companies': CompanyReadSerializer(Company.objects.all(), many=True).data
    }
    return render(request, 'microservices/add_microservice.html', context)

def redirect_to_main(request):
    body_unicode = request.body.decode('utf-8')
    dct = {pair.split('=')[0]: pair.split('=')[1] for pair in body_unicode.split('&')[:-1]}
    dct['author'] = Author.get_by_id(int(float(dct['author'])))
    dct['company'] = Company.get_by_id(int(float(dct['company'])))
    Microservice.objects.create(**dct)
    return index(request)

class MicroserviceViewSet(ModelViewSet):
    serializer_class = MicroserviceSerializer
    queryset = Microservice.objects.all().prefetch_related('author', 'company', 'tag_set')
    filter_backends = (DjangoFilterBackend, OrderingFilter)
    filterset_class = MicroserviceFilter
    ordering_fields = ('name', 'external_id')

    def list(self, request, *args, **kwargs):
        queryset = self.filter_queryset(self.get_queryset())

        if 'ordering' in request.GET:
            if request.GET['ordering'] in ['tags', '-tags']:
                tag_count = 'tag_count'
                if request.GET['ordering'] == '-tags':
                    tag_count = '-tag_count'
                queryset = queryset.annotate(tag_count=Count('tag')).order_by(tag_count)

        page = self.paginate_queryset(queryset)
        if page is not None:
            serializer = self.get_serializer(page, many=True)
            return self.get_paginated_response(serializer.data)

        serializer = self.get_serializer(queryset, many=True)
        return Response(serializer.data)

class AuthorViewSet(ModelViewSet):
    serializer_class = AuthorSerializer
    queryset = Author.objects.all()

    @action(detail=True, methods=['get'])
    def microservices(self, request, pk):
        author_obj = Author.get_by_id(pk)
        microservices = self.paginate_queryset(Microservice.objects.filter(author=author_obj))
        return self.get_paginated_response(data=MicroserviceReadSerializer(microservices, many=True).data)

class CompanyViewSet(ModelViewSet):
    serializer_class = CompanySerializer
    queryset = Company.objects.all()

    @action(detail=True, methods=['get'])
    def microservices(self, request, pk):
        company_obj = Company.get_by_id(pk)
        microservices = self.paginate_queryset(Microservice.objects.filter(company=company_obj))
        return self.get_paginated_response(data=MicroserviceReadSerializer(microservices, many=True).data)

class TagViewSet(ModelViewSet):
    serializer_class = TagSerializer
    queryset = Tag.objects.all()

class PairViewSet(ModelViewSet):
    serializer_class = PairSerializer
    queryset = Pair.objects.all()

    @action(detail=True, methods=['get'])
    def connections(self, request, pk):
        microservice_obj = Microservice.get_by_id(pk)
        pairs = Pair.objects.filter(
            Q(first_microservice=microservice_obj) | Q(second_microservice=microservice_obj)
        ).distinct()
        return Response({'main_node': microservice_obj.name,
                         'pairs': [[pair.first_microservice.name, pair.second_microservice.name] for pair in pairs]},
                        status=status.HTTP_200_OK)

    @action(detail=False, methods=['get'])
    def all_connections(self, request, pk=None):
        pairs = Pair.objects.all().distinct()
        return Response([[pair.first_microservice.name, pair.second_microservice.name] for pair in pairs],
                        status=status.HTTP_200_OK)