'''
This script is used for ADS model execution QC
Author: Cheng Qian
Date: 08/26/2017
'''

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import warnings
import gc
import sys
warnings.filterwarnings("ignore")

def describe_table(df):
    if df is None:
        return None
    else:
        df_input_dtypes = pd.DataFrame(df.dtypes,columns=['Variable Type'])
        df_input_dtypes = df_input_dtypes.reset_index()
        df_input_dtypes['Variable Name'] = df_input_dtypes['index']
        df_input_dtypes = df_input_dtypes[['Variable Name','Variable Type']]

        mis_val = df.isnull().sum()
        mis_val_percent = df.isnull().sum()/len(df)
        mis_val_table = pd.concat([mis_val, mis_val_percent], axis=1)
        mis_val_table[2]=list(map(lambda x: "{0:.2f}%".format(x * 100),mis_val_table[1]))
        mis_val_table[1]=list(map(lambda x: round(float(x),2),mis_val_table[1]))
        mis_val_table_ren_columns = mis_val_table.rename(
        columns = {0 : 'Number of Missing Value',
                   1 : 'Float of Missing Value',
                   2 : 'Percentage of Missing Value'}
                   )
        mis_val_table_ren_columns = mis_val_table_ren_columns.reset_index()
        mis_val_table_ren_columns['Variable Name'] = mis_val_table_ren_columns['index']
        mis_val_table_ren_columns = mis_val_table_ren_columns[['Variable Name','Number of Missing Value', 'Float of Missing Value','Percentage of Missing Value']]

        df_input_dtypes=df_input_dtypes.merge(mis_val_table_ren_columns, left_on='Variable Name', right_on='Variable Name',how='left')
        df_input_dtypes['Variable Type']=df_input_dtypes['Variable Type'].astype(str)
        df_input_dtypes.sort_values(['Float of Missing Value'],ascending=False,inplace=True)
        df_input_dtypes['Number of Records']=df.shape[0]
        df_input_dtypes.drop(['Float of Missing Value'],1,inplace=True)
    return df_input_dtypes

def data_QC(df,varlist,name):
    description1=pd.DataFrame()
    for col in varlist:
        t1=df[col].describe()
        t1=pd.DataFrame({'Index':t1.index,name:t1.values})
        t1=t1[1:]
        #t1.loc[-1]={'Index':'Skew',name:df[col].skew()}
        #t1.loc[-2]={'Index':'Kurtosis',name:df[col].kurtosis()}
        t1.loc[-1]={'Index':'%Missing',name:round((1-df[col].count().astype(float)/df.shape[0]),2)}
        t1.loc[-2]={'Index':'Variable Name',name:col}
        t1=t1.sort_index()
        description1 = pd.concat([description1,t1] ,axis=0)
    description1=pd.DataFrame(description1)
    description1.reset_index(inplace=True,drop=True)
    return description1

def Var_Check(df,varlist,name):
    if df is None:
        return True
    else:
        wrong=[]
        for col in varlist:
            if col not in df:
                print ('Variable %s not found in %s'%(col,name))
                wrong.append(col)

        if len(wrong)>0:
            print ('--------')
            return False
        else:
            return True

Score_Var_to_Check=[str(x).upper() for x in Score_Var_to_check]

def col_check(x):
    if x is not None:
        return str(x).upper()
    else:
        return None

MDL_Score=col_check(MDL_Score)
Data_Score=col_check(Data_Score)
MDL_Decile=col_check(MDL_Decile)
Data_Decile=col_check(Data_Decile)

'''#####################################################  Data Path Check ######################################################'''
Check=True
print ('Checking attributes and datasets...')

def load_data(path,message,chunksize=100000):
   if not os.path.exists(path):
       print (message)
       raise ValueError
       return None, False
   else:
       data_chunk=pd.read_csv(path,chunksize=chunksize)
       data=pd.DataFrame()
       data=pd.concat([chunk for chunk in data_chunk],ignore_index=True)
       data.columns=[str(x).upper() for x in data.columns]
       return data, True

message='Error: Model development data not found'
Mdl_Dev_data,Check	   = load_data(mdl_transform_path,message,chunksize=100000)
message='Error: Scoring data not found'
New_Scoring_data,Check = load_data(score_transform_path,message,chunksize=100000)

'''####################################################  Variable Check  ########################################################'''
if Check:
    check1=Var_Check(Mdl_Dev_data,   Score_Var_to_Check,'Modeling Development Datasets')
    check2=Var_Check(New_Scoring_data,Score_Var_to_Check,'New Scoring Datasets')
    if False in [check1,check2]:
        Check=False

score_check=True
if None in (MDL_Score,Data_Score):
    score_check=False

decile_check=True
if None in (MDL_Decile,Data_Decile):
    decile_check=False

if Check:

    print ('Data and Attributes read!')
    print ('')

    df_mdl_desc = describe_table(Mdl_Dev_data[Score_Var_to_Check])
    df_score_desc = describe_table(New_Scoring_data[Score_Var_to_Check])

    '''''################################################## Variable Classification ###############################################'''

    var_category={'Category':[],'Continuous':[]}

    var_category['Category']  =[col for col in list(Mdl_Dev_data.select_dtypes(include=['object']).columns) if col in Score_Var_to_Check]
    var_category['Continuous']=[col for col in list(Mdl_Dev_data.select_dtypes(exclude=['object']).columns) if col in Score_Var_to_Check]

    removeal=[]
    for column in var_category['Continuous']:
        unique_num=Mdl_Dev_data[column].nunique()
        if int(unique_num)<10:
            #print (column)
            var_category['Category'].append(column)
            removeal.append(column)
    #var_category['Continuous'].remove(column)
    continuous2=[col for col in var_category['Continuous'] if col not in removeal]
    var_category['Continuous']=continuous2
    '''''################################################## Transform&Scoring Data QC ###############################################'''

    print ('Writing descriptions of datasets...')
    continuous_check=[col for col in list(Mdl_Dev_data.select_dtypes(exclude=['object']).columns) if col in Score_Var_to_Check]
    description1=data_QC(Mdl_Dev_data,continuous_check,mdl_name)
    description2=data_QC(New_Scoring_data,continuous_check,scr_name)

    Transform_QC = description1.merge(description2,left_index=True,right_index=True, how='inner')[['Index_x',mdl_name,scr_name]]

    for index in Transform_QC.index:
        try:
            p=abs((Transform_QC.loc[index,scr_name]-Transform_QC.loc[index,mdl_name])/(Transform_QC.loc[index,mdl_name]+0.00001))
            Transform_QC.set_value(index,'Percentage Change',round(p,2))
        except:
            Transform_QC.set_value(index,'Percentage Change',0)
    Transform_QC.rename(columns={'Index_x':'Index'},inplace=True)

    if score_check:
        if decile_check:
            Mdl_Dev_data['MDL_Decile'] = Mdl_Dev_data[MDL_Decile]
            New_Scoring_data['Score_decile'] = New_Scoring_data[Data_Decile]
            decile_range=sorted(Mdl_Dev_data['MDL_Decile'].unique())
        else:
            Mdl_Dev_data['MDL_Decile'] = 10-pd.qcut(Mdl_Dev_data[MDL_Score], 10, labels=False)
            New_Scoring_data['Score_decile'] = 10-pd.qcut(New_Scoring_data[Data_Score], 10, labels=False)
            decile_range=range(1,11)

        print ('Checking model decile scores...')
        a=Mdl_Dev_data.groupby(['MDL_Decile'])[MDL_Score].min()
        b=Mdl_Dev_data.groupby(['MDL_Decile'])[MDL_Score].median()
        c=New_Scoring_data.groupby(['Score_decile'])[Data_Score].min()
        d=New_Scoring_data.groupby(['Score_decile'])[Data_Score].median()

        Score_QC=pd.concat([a,c,b,d] ,keys=[mdl_name+'_Min',scr_name+'_Min',mdl_name+'_Median',scr_name+'_Median'],axis=1)
        Score_QC['%Min_Change']=abs(Score_QC[mdl_name+'_Min']-Score_QC[scr_name+'_Min'])/Score_QC[mdl_name+'_Min']
        Score_QC['%Min_Change']=list(map(lambda x: "{0:.2f}%".format(x*100),Score_QC['%Min_Change']))

        Score_QC['%Median_Change']=abs(Score_QC[mdl_name+'_Median']-Score_QC[scr_name+'_Median'])/Score_QC[mdl_name+'_Median']
        Score_QC['%Median_Change']=list(map(lambda x: "{0:.2f}%".format(x*100),Score_QC['%Median_Change']))

        Score_QC['Decile']=decile_range
        Score_QC=Score_QC[['Decile',mdl_name+'_Min',scr_name+'_Min','%Min_Change',mdl_name+'_Median',scr_name+'_Median','%Median_Change']]

    '''''#####################################################PSI###############################################################'''

    print ('Checking PSI value...')

    columns=['Variable','Values','Actual','Expected','A-E','ln(A/E)','PSI']
    psi_result=pd.DataFrame(columns=columns)

    for variable in var_category['Category']:
        result=pd.DataFrame(columns=columns)
        expected=(Mdl_Dev_data[variable].value_counts(normalize=True,dropna=False)).sort_index().to_frame(name='Expected')
        actual=(New_Scoring_data[variable].value_counts(normalize=True,dropna=False)).sort_index().to_frame(name='Actual')

        data=actual.merge(expected,left_index=True,right_index=True,how='left')

        result['Values']=data.index
        result['Expected']=list(map(lambda x: round(x,2),data['Expected']))
        result['Actual']=list(map(lambda x: round(x,2),actual['Actual']))
        result['A-E']=list(map(lambda x: round(x,2),result['Actual']-result['Expected']))
        result['ln(A/E)']=list(map(lambda x: round(x,2),np.log(result['Actual']/result['Expected'])))
        result['PSI']=result['A-E'] * result['ln(A/E)']
        result['Variable']=variable
        variable_psi=np.nansum(list(filter(lambda x: x not in [np.Inf,np.nan],result['PSI'])))
        result.set_value(result.index.max()+1,'ln(A/E)','Sum')
        result.set_value(result.index.max(),'PSI',variable_psi)
        result.loc[result.index.max()+1]=np.NaN
        psi_result=pd.concat([psi_result,result],axis=0)

    for variable in var_category['Continuous']:
        result=pd.DataFrame(columns=columns)
        quantile_values=Mdl_Dev_data[variable].quantile(q=np.linspace(0.0,1.0,11)).values
        cutoff=sorted(list(set(quantile_values)))

        values,expected,actual=[],[],[]

        for index in range(len(cutoff)-1):
            if cutoff[index]<0.001:
                n=6
            else:
                n=2
            #minimum=round(cutoff[index],n)
            #maximum=round(cutoff[index+1],n)
            minimum=cutoff[index]
            maximum=cutoff[index+1]

            if index==0:
                values.append('<%s'%str(maximum))
                expected.append(float(Mdl_Dev_data[Mdl_Dev_data[variable]<maximum].shape[0]) / Mdl_Dev_data.shape[0])
                actual.append(float(New_Scoring_data[New_Scoring_data[variable]<maximum].shape[0]) / New_Scoring_data.shape[0])
            elif index<=len(cutoff)-3:
                values.append('%s - %s'%(str(minimum),str(maximum)))
                expected.append(float(Mdl_Dev_data[(Mdl_Dev_data[variable]>=minimum) & (Mdl_Dev_data[variable]<maximum)].shape[0]) / Mdl_Dev_data.shape[0])
                actual.append(float(New_Scoring_data[(New_Scoring_data[variable]>=minimum) & (New_Scoring_data[variable]<maximum)].shape[0]) / New_Scoring_data.shape[0])
            else:
                values.append('>=%s'%str(minimum))
                expected.append(float(Mdl_Dev_data[Mdl_Dev_data[variable]>=minimum].shape[0]) / Mdl_Dev_data.shape[0])
                actual.append(float(New_Scoring_data[New_Scoring_data[variable]>=minimum].shape[0]) / New_Scoring_data.shape[0])
        if Mdl_Dev_data[variable].isnull().sum()>0 or New_Scoring_data[variable].isnull().sum()>0:
            values.append('Missing')
            expected.append(float(Mdl_Dev_data[variable].isnull().sum()) / Mdl_Dev_data.shape[0])
            actual.append(float(New_Scoring_data[variable].isnull().sum()) / New_Scoring_data.shape[0])

        result['Values']=values
        result['Expected']=list(map(lambda x: round(x,2),expected))
        result['Actual']=list(map(lambda x: round(x,2),actual))
        result['A-E']=list(map(lambda x: round(x,2),result['Actual']-result['Expected']))
        result['ln(A/E)']=list(map(lambda x: round(x,2),np.log(result['Actual']/result['Expected'])))
        result['PSI']=result['A-E'] * result['ln(A/E)']
        result['Variable']=variable
        variable_psi=np.nansum(list(filter(lambda x: x not in [np.Inf,np.nan],result['PSI'])))
        result.set_value(result.index.max()+1,'ln(A/E)','Sum')
        result.set_value(result.index.max(),'PSI',variable_psi)
        result.loc[result.index.max()+1]=np.NaN
        psi_result=pd.concat([psi_result,result],axis=0)

    psi_result['PSI']=list(map(lambda x: round(x,3),psi_result['PSI']))

    psi_result.reset_index(inplace=True,drop=True)

    psi_result.rename(columns={'Actual':'Actual:'+scr_name,'Expected':'Expected:'+mdl_name},inplace=True)

    '''''#####################################################Write into Excel###############################################################'''

    print ('')
    print ('Writing into Excel...')
    writer = pd.ExcelWriter(output_path)
    workbook  = writer.book
    cell_format = workbook.add_format({'align': 'left',
                                       'valign': 'vcenter',
                                       'border': 1,
                                       'bold': False,
                                       'text_wrap': True})
    percentage_format = workbook.add_format({'num_format': '0.0%'})

    df_mdl_desc.to_excel(writer,'01-Data Overview',index=False)

    df_score_desc.to_excel(writer,'01-Data Overview',index=False,startrow=df_mdl_desc.shape[0]+3)
    worksheet=writer.sheets['01-Data Overview']

    text='Overview of %s'%mdl_name
    worksheet.merge_range('H2:M5', text, cell_format)

    text='Overview of %s' %scr_name
    worksheet.merge_range('H%i:M%i'%(df_mdl_desc.shape[0]+4,df_mdl_desc.shape[0]+7), text, cell_format)

    Transform_QC.to_excel(writer,'02-Data Attributes QC',index=False)
    text='''This Tab is to compare distribution after data transformation;\n
    Column B is {0} Value;\n
    Column C is New Scoring Datasets Value;\n
    Column D is Percentage Change,values higher than 20% will be highlighted red;'''.format(mdl_name)
    worksheet=writer.sheets['02-Data Attributes QC']
    worksheet.merge_range('F3:M11', text, cell_format)

    max_length=Transform_QC.shape[0]+1
    font_format = workbook.add_format({'font_color': 'red','num_format': '0.0%'})
    border_format = workbook.add_format({'border':1})
    QC_sheet='02-Data Attributes QC'

    worksheet=writer.sheets[QC_sheet]
    worksheet.conditional_format('D2:D%i'%max_length, {'type':     'cell',
                                                       'criteria': '>=',
                                                       'value':    0.0,
                                                       'format':   percentage_format})
    worksheet.conditional_format('D2:D%i'%max_length, {'type':     'cell',
                                                       'criteria': 'greater than',
                                                       'value':    0.2,
                                                       'format':   font_format})

    for row in range(2,Transform_QC.shape[0],9):
        worksheet.conditional_format('A%i:D%i'%(row,row),{'type':     'cell',
                                                          'criteria': '>=',
                                                          'value':    0,
                                                          'format':  border_format})

    '''Score Decile plot'''
    if score_check:
        Score_QC=Score_QC[['Decile',mdl_name+'_Min',scr_name+'_Min','%Min_Change',mdl_name+'_Median',scr_name+'_Median','%Median_Change']]
        Score_QC.to_excel(writer,'03-Score QC Plot',index=False)
        position=len(decile_range)+1

        worksheet = writer.sheets['03-Score QC Plot']

        chart = workbook.add_chart({'type': 'column'})
        chart.add_series({'values': '''='03-Score QC Plot'!$B$2:$B$%i'''%position,'name':'%s Score'%mdl_name})
        chart.add_series({'values': '''='03-Score QC Plot'!$C$2:$C$%i'''%position,'name':'%s Score'%scr_name})

        chart.set_x_axis({'name': 'Model Decile'})
        chart.set_title({'name':'Minimum Score by Decile Comparison','name_font': {'size': 12}})

        worksheet.insert_chart('B13', chart)

    '''PSI'''

    psi_result.to_excel(writer,'04-PSI',index=False)
    worksheet = writer.sheets['04-PSI']
    font_format = workbook.add_format({'font_color': 'red'})
    worksheet.conditional_format('G2:G%i'%(psi_result.shape[0]+1),{'type':     'cell',
                                                      'criteria': '>=',
                                                      'value':    0.1,
                                                      'format':  font_format})

    text=''' This tab is to calculate PSI(Population Stability Index) of each variable \n
PSI<0.1 : Minor Shift \n
PSI Between 0.1 and 0.25: Some Minor Change \n
PSI>0.25 : Major Shift \n

Actual: Percentage of customers by range in the %s \n
Expected: Percentage of customers by range in the %s \n
'''    %(scr_name,mdl_name)
    worksheet.merge_range('J6:P20', text, cell_format)

    '''Univariate Analysis'''
    def Decile_Avg_Mid(var_name, df, decile_name):
        avg = df.groupby(decile_name)[var_name].mean().to_frame(name='Average')
        mid = df.groupby(decile_name)[var_name].median().to_frame(name='Median')

        result = avg.merge(mid,left_index=True,right_index=True,how='inner')

        return result

    numeric=list(Mdl_Dev_data.select_dtypes(exclude=['object']).columns)
    univariate_check_cols=[col for col in numeric if col in Score_Var_to_Check]

    for index_num, col in enumerate(univariate_check_cols):
        mdl_analysis	= Decile_Avg_Mid(col, Mdl_Dev_data,     'MDL_Decile')
        scoring_analysis= Decile_Avg_Mid(col, New_Scoring_data, 'Score_decile')
        result = pd.concat([mdl_analysis, scoring_analysis], keys=[mdl_name, scr_name],axis=1)
        result.index.name = col
        result.to_excel(writer,'04-Univariate Analysis',index=True,startrow=(result.shape[0]+8)*index_num,startcol=0)

        start=(result.shape[0]+8)*index_num+4
        end=(result.shape[0]+8)*index_num+13
        chart = workbook.add_chart({'type': 'column'})
        chart.add_series({'values': '''='04-Univariate Analysis'!$B$%i:$B$%i'''%(start,end),'name':'%s Average Score'%mdl_name})
        chart.add_series({'values': '''='04-Univariate Analysis'!$D$%i:$D$%i'''%(start,end),'name':'%s Average Score'%scr_name})

        chart.set_x_axis({'name': 'Model Decile'})
        chart.set_title({'name':'%s Average by Decile'%col,'name_font': {'size': 12}})

        chart.set_legend({'font': {'size': 9},'position': 'top'})
        writer.sheets['04-Univariate Analysis'].insert_chart('I%i'%(start-3), chart)

    #formater = workbook.add_format({'border':0})
    #writer.sheets['04-Univariate Analysis'].set_column('A:DC',None,formater)
    writer.sheets['04-Univariate Analysis'].hide_gridlines(2)
    writer.sheets['03-Score QC Plot'].hide_gridlines(2)
    writer.sheets['02-Data Attributes QC'].hide_gridlines(2)
    writer.sheets['01-Data Overview'].hide_gridlines(2)

    writer.save()
    writer.close()

    print ('')
    print ('Complete! QC report is here! %s'%output_path)
    del Mdl_Dev_data,New_Scoring_data
    gc.collect()


#DESAFIO064 Crie um programa que leia vários números inteiros pelo telcado. O programa só vai parar qual o usuário digitar o valor 999, que é a condição de parada. No final, mostre quantos números foram digitados e qual foi a soma entre eles (desconsiderando o flag).

n = 0
c = 0
s = 0
while n != 999:
    n = int(input('Digite um número: '))
    if n == 999:
        break
    s += n
    c += 1
print('A soma dos números digitados é {} e você digitou {} números.'.format(s, c))


#!/usr/local/bin/python3

import argparse
from fontTools.ttLib import TTFont

# Parse the command-line arguments
parser = argparse.ArgumentParser()
parser.add_argument('font_path', help='Path to the font file')
args = parser.parse_args()

# Open the font file
font = TTFont(args.font_path)

# Get the list of Unicode characters that are available in the font
characters = font['cmap'].getBestCmap().keys()

# Print the list of characters
for character in characters:
    print(format(character, "x"), end=",")


from typing import Iterable, Tuple

from platform import system
import psutil

from myconky.core.receipt import AbstractRepecit
from myconky.core.utils import gauge, bytes_fmt

class DiskSpaceRecepit(AbstractRepecit):
    def get_info(self) -> Iterable[Tuple[str, str]]:
        for disk in psutil.disk_partitions(all=False):
            if system() == 'Darwin':
                if disk.mountpoint == '/private/var/vm':
                    continue

            usage = psutil.disk_usage(disk.mountpoint)

            if usage.total != 0:
                usage_perc = (usage.used/usage.total) * 100

                yield (
                    disk.mountpoint,
                    gauge(usage_perc, summary="%s/%s" % (bytes_fmt(usage.used), bytes_fmt(usage.total))),
                )


#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Optimize confusion matrix.

For more information, see

* http://cs.stackexchange.com/q/70627/2914
* http://datascience.stackexchange.com/q/17079/8820
"""

import json
import numpy as np
import matplotlib.pyplot as plt
import random
random.seed(0)
import logging
import sys
import csv
import os

logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
                    level=logging.DEBUG,
                    stream=sys.stdout)

def read_symbols(symbol_file='symbols.csv'):
    """Read symbols."""
    with open(symbol_file) as fp:
        reader = csv.reader(fp, delimiter=',', quotechar='"')
        data_read = [row for row in reader]
    return [el[1] for el in data_read]

def calculate_score(cm):
    """Calculate a score how close big elements of cm are to the diagonal."""
    score = 0
    for i, line in enumerate(cm):
        for j, el in enumerate(line):
            score += el * abs(i - j)
    return score

def swap(cm, i, j):
    """Swap row and column i and j."""
    # swap columns
    copy = cm[:, i].copy()
    cm[:, i] = cm[:, j]
    cm[:, j] = copy
    # swap rows
    copy = cm[i, :].copy()
    cm[i, :] = cm[j, :]
    cm[j, :] = copy
    return cm

def swap_1d(perm, i, j):
    """Swap two elements of a 1-D numpy array in-place."""
    perm[i], perm[j] = perm[j], perm[i]
    return perm

def apply_permutation(cm, perm):
    """Apply permutation to a matrix."""
    return cm[perm].transpose()[perm].transpose()

def simulated_annealing(current_cm,
                        current_perm=None,
                        steps=2 * 10**5,
                        temp=100.0,
                        cooling_factor=0.99,
                        deterministic=False):
    """
    Optimize current_cm by randomly swapping elements.

    Parameters
    ----------
    current_cm : numpy array
    current_perm : None or iterable, optional (default: None)
    steps : int, optional (default: 2 * 10**4)
    temp : float > 0.0, optional (default: 100.0)
        Temperature
    cooling_factor: float in (0, 1), optional (default: 0.99)
    """
    assert temp > 0
    assert cooling_factor > 0
    assert cooling_factor < 1
    n = len(current_cm)
    if current_perm is None:
        current_perm = list(range(n))
    current_perm = np.array(current_perm)
    current_cm = apply_permutation(current_cm, current_perm)
    current_score = calculate_score(current_cm)
    best_perm = current_perm
    best_cm = current_cm
    best_score = current_score
    print("Starting Score: {}".format(current_score))
    for _ in range(steps):
        tmp = np.array(current_cm, copy=True)
        i = random.randint(0, n - 1)
        j = random.randint(0, n - 1)
        perm = swap_1d(current_perm.copy(), i, j)
        tmp = swap(tmp, i, j)
        # tmp = apply_permutation(tmp, perm)
        tmp_score = calculate_score(tmp)
        if deterministic:
            chance = 1.0
        else:
            chance = random.random()
            temp *= 0.99
        hot_prob = min(1, np.exp(-(tmp_score - current_score) / temp))
        if chance <= hot_prob:
            if best_score > tmp_score:
                best_perm = perm
                best_cm = tmp
                best_score = tmp_score
            current_score = tmp_score
            current_cm = tmp
            logging.info("Current: %i (best: %i, hot_prob=%0.4f%%)",
                         current_score,
                         best_score,
                         (hot_prob * 100))
    return {'cm': best_cm, 'perm': best_perm}

def plot_cm(cm, zero_diagonal=False):
    """Plot a confusion matrix."""
    if zero_diagonal:
        for i in range(len(cm)):
            cm[i][i] = 0  # Set diagonal to 0 to make other stuff visible

    norm_conf = np.zeros(cm.shape)
    for i, line in enumerate(cm):  # line
        a = max(sum(line), 10**-7)
        for j, el in enumerate(line):  # el
            norm_conf[i][j] = (float(el) / float(a))

    n = len(cm)
    size = int(n / 4.)
    fig = plt.figure(figsize=(size, size), dpi=80, )
    plt.clf()
    ax = fig.add_subplot(111)
    ax.set_aspect(1)
    res = ax.imshow(np.array(norm_conf), cmap=plt.cm.viridis,
                    interpolation='nearest')

    width, height = cm.shape

    # for x in xrange(width):
    #     for y in xrange(height):
    #         ax.annotate(str(cm[x][y]), xy=(y, x),
    #                     horizontalalignment='center',
    #                     verticalalignment='center')

    fig.colorbar(res)
    # alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
    # plt.xticks(range(width), alphabet[:width])
    # plt.yticks(range(height), alphabet[:height])
    plt.savefig('confusion_matrix.png', format='png')

def main(perm_file, steps):
    """
    Run optimization and generate output.
    """
    with open('confusion-matrix.json') as f:
        cm = json.load(f)
        cm = np.array(cm)

    # Visulize
    print("Score: {}".format(calculate_score(cm)))

    if os.path.isfile(perm_file):
        with open(perm_file) as data_file:
            perm = json.load(data_file)
        print(perm)
    else:
        perm = list(range(len(cm)))
    result = simulated_annealing(cm, perm, deterministic=True, steps=steps)
    print("Score: {}".format(calculate_score(result['cm'])))
    print("Perm: {}".format(list(result['perm'])))
    symbols = read_symbols()
    print("Symbols: {}".format([symbols[i] for i in perm]))
    plot_cm(result['cm'], zero_diagonal=True)

def get_parser():
    """Get parser object for script xy.py."""
    from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
    parser = ArgumentParser(description=__doc__,
                            formatter_class=ArgumentDefaultsHelpFormatter)
    parser.add_argument("--perm",
                        dest="perm_file",
                        help=("path of a json file with a permutation to "
                              "start with"),
                        metavar="perm.json",
                        default="")
    parser.add_argument("-n",
                        dest="n",
                        default=4 * 10**5,
                        type=int,
                        help="number of steps to iterate")
    return parser

if __name__ == "__main__":
    args = get_parser().parse_args()
    main(args.perm_file, args.n)


import numpy as np
from astropy.time import Time
from astropy import units as u

from ...utils import _checkTime
from ...utils import getHorizonsVectors
from ...utils import getHorizonsEphemeris
from ...testing import testEphemeris
from ..orbits import Orbits
from ..ephemeris import generateEphemeris

TARGETS = [
    "Amor",
    "Eros",
    "Eugenia",
]
EPOCH = 57257.0
DT = np.array([0])
T0 = Time(
    [EPOCH],
    format="mjd",
    scale="tdb",
)
T1 = Time(
    EPOCH + DT,
    format="mjd",
    scale="tdb"
)
OBSERVATORIES = ["I11", "I41", "005", "F51", "500", "568", "W84", "012", "I40", "286"]
OBSERVERS =  {k:T1 for k in OBSERVATORIES}

def test_generateEphemeris():
    """
    Query Horizons (via astroquery) for initial state vectors of each target at T0, also query Horizons
    for ephemerides for each target as observed by each observatory at T1. Use THOR and PYOORB backend to
    generate ephemerides for each target as observed by each observatory at T1 using the initial state vectors.
    Compare the resulting ephemerides and test how well they agree with the ones pulled from Horizons.
    """
    # Query Horizons for ephemerides of each target as observed
    # by each observatory
    horizons_ephemeris = getHorizonsEphemeris(
        TARGETS,
        OBSERVERS
    )
    horizons_ephemeris = horizons_ephemeris[["RA", "DEC"]].values

    # Query Horizons for initial state vectors for each target at T0
    horizons_orbits = Orbits.fromHorizons(
        TARGETS,
        T0,
    )

    # Use PYOORB to generate ephemeris for each target observed by
    # each observer
    pyoorb_ephemeris = generateEphemeris(
        horizons_orbits,
        OBSERVERS,
        backend="PYOORB",
    )
    pyoorb_ephemeris = pyoorb_ephemeris[["RA_deg", "Dec_deg"]].values

    # pyoorb's ephemerides agree with Horizons' ephemerides
    # to within the tolerance below.
    testEphemeris(
        pyoorb_ephemeris,
        horizons_ephemeris,
        angle_tol=(50*u.milliarcsecond),
        magnitude=True
    )

    # Use MJOLNIR's 2-body propagator to generate ephemeris for each target observed by
    # each observer
    mjolnir_ephemeris = generateEphemeris(
        horizons_orbits,
        OBSERVERS,
        backend="MJOLNIR",
    )
    mjolnir_ephemeris = mjolnir_ephemeris[["RA_deg", "Dec_deg"]].values

    # MJOLNIR's 2-body ephemerides agree with Horizons' ephemerides
    # to within the tolerance below.
    testEphemeris(
        mjolnir_ephemeris,
        horizons_ephemeris,
        angle_tol=(50*u.milliarcsecond),
        magnitude=True
    )
    return


import datetime
import logging
import sys
import time
import traceback
from functools import wraps
from socket import error as SocketError

import praw.exceptions
import prawcore

def handle_api_exceptions(max_attempts=1):
    """Return a function decorator that wraps a given function in a
    try-except block that will handle various exceptions that may
    occur during an API request to reddit. A maximum number of retry
    attempts may be specified.
    """

    def decorator(func):
        @wraps(func)
        def wrapper(*args, **kwargs):
            retries = 0
            while retries < max_attempts:
                sleep_time = None
                error_msg = ""
                try:
                    return func(*args, **kwargs)
                # Handle and log miscellaneous API exceptions
                except praw.exceptions.PRAWException as e:
                    error_msg = "PRAW Exception \"{error}\" occurred: ".format(
                        error=e)
                except praw.exceptions.ClientException as e:
                    error_msg = "Client Exception \"{error}\" occurred: ".format(
                        error=e)
                except praw.exceptions.APIException as e:
                    error_msg = "API Exception \"{error}\" occurred: ".format(
                        error=e)
                except SocketError as e:
                    error_msg = "SocketError \"{error}\" occurred: ".format(
                        error=e)
                    args[0].log.error(error_msg)
                sleep_time = sleep_time or retries * 15
                args[0].log.error("{0} in {f}. Sleeping for {t} seconds. "
                               "Attempt {rt} of {at}.".format(error_msg, f=func.__name__,
                                                              t=sleep_time, rt=retries + 1, at=max_attempts))
                time.sleep(sleep_time)
                retries += 1

        return wrapper

    return decorator

class CommentsScanner:
    def __init__(self, db, reddit_client, wallet_id, rest_wallet, subreddit):
        self.wallet_id = wallet_id
        self.db = db
        self.reddit_client = reddit_client
        self.rest_wallet = rest_wallet
        self.subreddit = subreddit

        log_file_name = "comments_scanner_" + str(datetime.datetime.now().isoformat()) + ".log"
        logging.basicConfig(filename=log_file_name, level=logging.INFO, format='%(asctime)s %(message)s')
        log = logging.getLogger("comments")
        self.log = log

    @handle_api_exceptions(max_attempts=3)
    def comment_reply(self, comment, reply_text):
        self.log.info("COMMENT REPLY")
        self.log.info(reply_text)
        comment.reply(reply_text)

    def send_tip(self, comment, amount, sender_user_address, receiving_address, receiving_user):
        try:
            self.log.info("Sending amount: " + str(amount))
            data = {'action': 'account_balance',
                    'account': sender_user_address}
            post_body = self.rest_wallet.post_to_wallet(data, self.log)
            data = {'action': 'rai_from_raw', 'amount': int(
                post_body['balance'])}
            rai_balance = self.rest_wallet.post_to_wallet(data, self.log)

            # float of total send
            rai_send = float(amount) * 1000000
            raw_send = str(int(rai_send)) + '000000000000000000000000'
            self.log.info("Current rai balance: " + str(rai_balance['amount']))
            # check amount left
            if int(rai_send) <= int(rai_balance['amount']):
                self.log.info('Tipping now')
                data = {'action': 'send', 'wallet': self.wallet_id, 'source': sender_user_address,
                        'destination': receiving_address, 'amount': int(raw_send)}
                post_body = self.rest_wallet.post_to_wallet(data, self.log)
                reply_text = 'Tipped %s to /u/%s\n\nBlock: %s' % (
                    amount, receiving_user, str(post_body['block']))
            else:
                reply_text = 'Not enough in your account to tip'

            self.comment_reply(comment, reply_text)
        except TypeError as e:
            tb = traceback.format_exc()
            self.log.error(e)
            self.log.error(tb)
        except:
            self.log.error("Unexpected error in send_tip: " + str(sys.exc_info()[0]))
            tb = traceback.format_exc()
            self.log.error(tb)

    def process_tip(self, amount, comment, receiving_user):
        user_table = self.db['user']
        comment_table = self.db['comments']

        # See if we have an author xrb address and a to xrb address, if not invite to register
        self.log.info('Looking for sender in db')
        if user_table.find_one(user_id=comment.author.name):
            self.log.info('Sender in db')
            # Author registered
            sender_user_data = user_table.find_one(
                user_id=comment.author.name)
            sender_user_address = sender_user_data['xrb_address']

            if user_table.find_one(user_id=receiving_user):
                user_data = user_table.find_one(
                    user_id=receiving_user)
                receiving_address = user_data['xrb_address']
            else:
                self.log.info('Not in DB - registering')
                # Generate address
                data = {'action': 'account_create',
                        'wallet': self.wallet_id}
                post_body = self.rest_wallet.post_to_wallet(data, self.log)
                self.log.info(post_body['account'])

                # Add to database
                record = dict(user_id=receiving_user, xrb_address=post_body['account'])
                self.log.info("Inserting into db: " + str(record))
                user_table.insert(record)
                receiving_address = post_body['account']
                reply_text = str(receiving_user) + ' isnt registered with the bot so Ive made an account for them, ' \
                             + 'they can access it by DM the bot'
                try:
                    self.comment_reply(comment, reply_text)
                except:
                    self.log.error("Unexpected error in process_tip: " + str(sys.exc_info()[0]))
                    tb = traceback.format_exc()
                    self.log.error(tb)

            self.send_tip(comment, amount, sender_user_address, receiving_address, receiving_user)

        else:
            reply_text = 'Hi, /u/' + str(comment.author.name) + ' please register with the bot by sending it a' \
                         + ' message and it will make you an account'
            self.comment_reply(comment, reply_text)

        # Add to db
        record = dict(
            comment_id=comment.fullname, to=receiving_user, amount=amount, author=comment.author.name)
        self.log.info("Inserting into db: " + str(record))
        comment_table.insert(record)
        self.log.info('DB updated')

    def parse_tip(self, comment, parts_of_comment):
        # get a reference to the table 'comments'
        comment_table = self.db['comments']

        # print(vars(comment))

        # Save the comment id in a database so we don't repeat this
        if comment_table.find_one(comment_id=comment.fullname):
            self.log.info('Already in db, ignore')
        else:
            length = len(parts_of_comment)
            if (length == 2) or (length == 3):
                # default to 0 send and overwrite with input
                amount = 0
                receiving_user = ''
                if length == 2:
                    amount = parts_of_comment[1]
                    receiving_user = comment.link_author

                elif length == 3:
                    split_user = parts_of_comment[1]
                    amount = parts_of_comment[2]
                    if split_user[:3] == '/u/':
                        receiving_user = split_user[3:]
                    else:
                        receiving_user = split_user

                self.log.info(receiving_user)
                self.process_tip(amount, comment, receiving_user)
            else:
                self.log.info('Invalid amount')
                self.comment_reply(comment, 'Tip command is invalid. Follow the format `!tipxrb <username> <amount>`')

    def parse_comment(self, comment):
        parts_of_comment = comment.body.split(" ")

        if parts_of_comment[0].lower() == '!tipxrb':
            self.log.info('\n\n')
            self.log.info('Found tip reference in comments')
            self.parse_tip(comment, parts_of_comment)

    def scan_comments(self):
        subreddit_client_ = self.reddit_client.subreddit(self.subreddit)

        self.log.info('Tracking r/' + self.subreddit + ' Comments')

        try:
            for comment in subreddit_client_.stream.comments():
                self.parse_comment(comment)

        except (praw.exceptions.PRAWException, prawcore.exceptions.PrawcoreException) as e:
            tb = traceback.format_exc()
            self.log.error("could not log in because: " + str(e))
            self.log.error(tb)

    def run_scan_loop(self):
        while 1:
            self.scan_comments()


from matplotlib import pyplot as plt
from matplotlib_venn import venn2
plt.figure(figsize=(4,4))
v = venn2(subsets = (1, 1, 0.5), set_labels=('','', '',), set_colors=('red', 'blue', 'green'), alpha=0.8)
plt.plot(-0.5,0.2,'')
plt.text(-0.55,0.0, 'Inactive rhodopsin')
plt.text(-0.12,0.0, 'Common contacts')
plt.text(0.157,0.2, 'Unique contacts in other states:')
plt.text(0.22,0.1, 'Bathorhodopsin(1)')
plt.text(0.22,0.0, 'Lumirhodopsin(1)')
plt.text(0.22,-0.1, 'Metarhodopsin(3)')
plt.text(0.22,-0.2, 'M_Gα(3)')
plt.text(0.22,-0.3, 'M_arrestin(1)')
plt.show()


from tkinter import *

screen = Tk()
screen.title("canvas Management")
screen.geometry("400x100")
screen.wm_minsize(width=300, height=300)

c1 = Canvas(screen, bg='gray')
c1.place(x=10, y=10, width=400, height=400)
c1.create_line(10,10,200,10, fill='yellow')
c1.create_rectangle(20,20,100,100, fill ='green', outline='red')
c1.create_oval(30,220,90,280, fill='navy', outline='white',width=2)
screen.mainloop()


import seaborn as sns
import matplotlib.pyplot as plt

def find_correlated_columns(df, target):
    # find correlation between columns
    corr = df.corr()
    # make values absolute and sort
    corr = abs(corr.sort_values(by=[target], ascending=False))

    # plot corr
    plt.figure(figsize=(10, 10))
    sns.heatmap(corr.round(2), annot=True, cmap=plt.cm.Reds)
    plt.show()

    # select highly correlated features
    corr_target = corr[target]

    relevant_features = corr_target[corr_target > 0.1]

    print(relevant_features)

    list_columns = []

    for col, val in relevant_features.iteritems():
        if val < 1:
            list_columns.append(col)

    return list_columns


from functools import partial
from itertools import cycle
英 = r'C:\Users\wuyudi\Desktop\temp1.txt'
中 = r'C:\Users\wuyudi\Desktop\temp2.txt'
输出 = r'C:\Users\wuyudi\Desktop\34 - 副本.txt'

Open = partial(open, encoding='utf-8')
with Open(中) as 中文文件, Open(英) as 英文文件, Open(输出, 'w') as f:
    for i, 英文, 中文 in zip(cycle(range(4)), 英文文件, 中文文件):
        f.write(英文)
        if i == 2:
            f.write(中文)

翻译工具/翻译工具.py


#-------------------------------------------------------------------------------
#
#       Module:         mapMakePrism.py
#       Subsystem:      mapMakePrism
#       Program:        mapMakePrism
#
#       Copyright (c) 1998 - Descartes Systems Sciences
#
#-------------------------------------------------------------------------------
#
#       Description:
#
#-------------------------------------------------------------------------------
#
#   $History: mapmakeprism $
#
# 1999-04-15 Decker; Added functionality to create pie-slices, and face-sharing
#            of top and bottom face, if pie-slices are choosen.
# 2000-04-29 Decker; Added functionality to create cylinder-walls, with indent.
#            Loads/Saves used settings in SETUP.QRK
#
#-------------------------------------------------------------------------------

#$Header$

Info = {
   "plug-in":       "Make n sided Prism",
   "desc":          "Make an n sided prism from user supplied specs",
   "date":          "29 apr 2000",
   "author":        "Tim Smith and Decker",
   "author e-mail": "etsmith@mindspring.com",
   "quark":         "Version 6.0" }

import math
import quarkx
import quarkpy.qmacro
import quarkpy.qtoolbar
import quarkpy.mapsearch
import quarkpy.mapbtns
from quarkpy.maputils import *

class MakePrismDlg (quarkpy.qmacro.dialogbox):

    #
    # dialog layout
    #

    size = (290, 334)
    dfsep = 0.4        # separation at 40% between labels and edit boxes
    dlgflags = FWF_KEEPFOCUS

    dlgdef = """
    {
        Style = "15"
	    Caption = "Make an N-Sided Prism"

	    tex: =
	    {
		Txt = "Prism texture :"
		Typ = "ET"
		SelectMe = "1"
	    }

	    vertex: =
	    {
		Typ = "X"
		Txt = "Start prism..."
		Cap = "on a vertex"
	    }

	    sides: =
	    {
		Txt = "Number of sides"
		Typ = "EF1"
		Min = '5'
	    }

	    radius: =
	    {
		Txt = "Radius of prism"
		Typ = "EF1"
	    }

	    height: =
	    {
		Txt = "Height of prism"
		Typ = "EF1"
	    }

	    snapgrid: =
	    {
		Typ = "X"
		Txt = "Vertex on grid"
		Cap = ""
	    }

	    gridsize: =
	    {
		Txt = "Size of grid"
		Typ = "EF1"
		Min = '1'
	    }

	    slice: =
	    {
		Typ = "X"
		Txt = "Pie-slices"
		Cap = "Yes, make me some slices"
	    }

	    shareface: =
	    {
		Typ = "X"
		Txt = "Share Faces"
		Cap = "Yes, Top and Bottom"
        Hint="Usefull for Pie-Slices, marking this will create the top and bottom"$0D
             "faces shared among the pie-slices."
	    }

	    sep: = { Typ = "S" }

        walls: =
	    {
		Typ = "X"
        Txt = "Wall Slices"
        Cap = "Yes, give me walls"
        Hint="Works only if Pie-Slices is marked."$0D
             "Remember to specify the wall thickness too."
	    }

        wallthickness: =
	    {
        Typ = "EF1"
        Txt = "Wall thickness"
        Min = '1'
	    }

        uprindent: =
	    {
        Txt = "Upper Indent"
		Typ = "EF1"
        Hint="To make slanted walls, specify the distance in units, positive or negative."$0D
             "Set the 'size of grid' to 1 for best results."
	    }

	    sep: = { Typ = "S" }

        MakePrism:py = { }

	    close:py = { }
	}
    """

    #
    # __init__ initialize the object
    #

    def __init__(self, form, editor):

        #
        # General initialization of some local values
        #

        self.editor = editor
        self.sellist = self.editor.visualselection ()

        #
        # Create the data source
        #

        src = quarkx.newobj (":")

        #
        # Based on the textures in the selections, initialize the
        # from and to textures
        #

        texlist = quarkx.texturesof (editor.layout.explorer.sellist)
        if len (texlist) ==1:
            src ["tex"] = texlist [0]
        else:
            src ["tex"] = quarkx.setupsubset()["TextureDef"]

        #
        # Populate the other values
        #

        if (quarkx.setupsubset(SS_MAP, "Options")["MakePrism_Sides"] is None):
            src ["vertex"] = "X"
            src ["sides"] = 6,
            src ["radius"] = 64,
            src ["height"] = 16,
            src ["snapgrid"] = "X"
            src ["gridsize"] = 4,
            src ["slice"] = ""
            src ["shareface"] = ""
            src ["walls"] = ""
            src ["wallthickness"] = 16,
            src ["uprindent"] = 0,
        else:
            src ["vertex"]        = quarkx.setupsubset(SS_MAP, "Options")["MakePrism_Vertex"]
            src ["sides"]         = quarkx.setupsubset(SS_MAP, "Options")["MakePrism_Sides"]
            src ["radius"]        = quarkx.setupsubset(SS_MAP, "Options")["MakePrism_Radius"]
            src ["height"]        = quarkx.setupsubset(SS_MAP, "Options")["MakePrism_Height"]
            src ["snapgrid"]      = quarkx.setupsubset(SS_MAP, "Options")["MakePrism_Snapgrid"]
            src ["gridsize"]      = quarkx.setupsubset(SS_MAP, "Options")["MakePrism_Gridsize"]
            src ["slice"]         = quarkx.setupsubset(SS_MAP, "Options")["MakePrism_Slice"]
            src ["shareface"]     = quarkx.setupsubset(SS_MAP, "Options")["MakePrism_Shareface"]
            src ["walls"]         = quarkx.setupsubset(SS_MAP, "Options")["MakePrism_Walls"]
            src ["wallthickness"] = quarkx.setupsubset(SS_MAP, "Options")["MakePrism_Wallthickness"]
            src ["uprindent"]     = quarkx.setupsubset(SS_MAP, "Options")["MakePrism_Upperindent"]

        #
        # Create the dialog form and the buttons
        #

        quarkpy.qmacro.dialogbox.__init__(self, form, src,
        close = quarkpy.qtoolbar.button(
            self.close,
            "close this box",
            ico_editor, 0,
            "Close"),
        MakePrism = quarkpy.qtoolbar.button(
            self.MakePrism,
            "make prism",
            ico_editor, 3,
            "Make Prism"))

    #
    # Perform the function
    #

    def MakePrism(self, btn):

        #
        # commit any pending changes in the dialog box
        #

        quarkx.globalaccept ()

        #
        # do the changes
        #

        #undo = quarkx.action ()

        #
        # Gather information about what is to be created
        #

        value = self.src ["radius"]
        radius = value [0]
        value = self.src ["height"]
        height = value [0] / 2
        value = self.src ["sides"]
        sides = value [0]
        value = self.src ["gridsize"]
        gridsize = value [0]
        tex = self.src ["tex"]
        snapgrid = self.src ["snapgrid"] is not None
        slice = self.src["slice"] is not None
        shareface = self.src["shareface"] is not None
        walls = self.src["walls"] is not None
        value = self.src["wallthickness"]
        wallthickness = value [0]
        value = self.src["uprindent"]
        uprindent = value [0]

        # Save the settings...
        quarkx.setupsubset(SS_MAP, "Options")["MakePrism_Vertex"]        =  self.src["vertex"]
        quarkx.setupsubset(SS_MAP, "Options")["MakePrism_Sides"]         =  self.src["sides"]
        quarkx.setupsubset(SS_MAP, "Options")["MakePrism_Radius"]        =  self.src["radius"]
        quarkx.setupsubset(SS_MAP, "Options")["MakePrism_Height"]        =  self.src["height"]
        quarkx.setupsubset(SS_MAP, "Options")["MakePrism_Snapgrid"]      =  self.src["snapgrid"]
        quarkx.setupsubset(SS_MAP, "Options")["MakePrism_Gridsize"]      =  self.src["gridsize"]
        quarkx.setupsubset(SS_MAP, "Options")["MakePrism_Slice"]         =  self.src["slice"]
        quarkx.setupsubset(SS_MAP, "Options")["MakePrism_Shareface"]     =  self.src["shareface"]
        quarkx.setupsubset(SS_MAP, "Options")["MakePrism_Walls"]         =  self.src["walls"]
        quarkx.setupsubset(SS_MAP, "Options")["MakePrism_Wallthickness"] =  self.src["wallthickness"]
        quarkx.setupsubset(SS_MAP, "Options")["MakePrism_Upperindent"]   =  self.src["uprindent"]

        # Create the prism
        if slice:
            p = quarkx.newobj ("n-prism:g");
        else:
            p = quarkx.newobj ("n-prism:p");

        # Create the top and bottom face
        if (slice and shareface) or (not slice):
            f = quarkx.newobj ("up:f")
            f ["v"] = (-radius, -radius, height, 128 - radius, -radius, height, -radius, 128 - radius, height)
            f ["tex"] = tex
            p.appenditem (f)

            f = quarkx.newobj ("down:f")
            f ["v"] = (-radius, -radius, -height, -radius, 128 - radius, -height, 128 - radius, -radius, -height)
            f ["m"] = "1"
            f ["tex"] = tex
            p.appenditem (f)

        # Create the sides
        angle = 3.14159 / 2
        step = (3.14159 * 2) / sides
        if (self.src ["vertex"] is None):
            angle = angle - step / 2

        i = 0
        while i < sides:

            # Compute the information about the side
            v1 = self.ComputePoint (angle, radius, gridsize, snapgrid, -height)
            v3 = self.ComputePoint (angle - step, radius, gridsize, snapgrid, -height)
            v2 = self.ComputePoint (angle, radius - uprindent, gridsize, snapgrid, height)

            if slice:
                p1 = quarkx.newobj ("prism-slice:p");

            # Create the face
            f = quarkx.newobj ("side:f")
            f ["v"] = (v1.x, v1.y, v1.z, v2.x, v2.y, v2.z, v3.x, v3.y, v3.z)
            f ["tex"] = tex
            if not slice:
                p.appenditem (f)
            else:
                # We want slices...
                p1.appenditem(f)

                f = quarkx.newobj ("side:f")
                f ["v"] = (0,0,0, v1.x,v1.y,v1.z+128, v1.x,v1.y,v1.z)
                f ["tex"] = tex
                p1.appenditem (f)

                f = quarkx.newobj ("side:f")
                f ["v"] = (0,0,0, v3.x,v3.y,v3.z, v3.x,v3.y,v3.z+128)
                f ["tex"] = tex
                p1.appenditem (f)

                # We want individual top/bottom faces...
                if not shareface:
                    f = quarkx.newobj ("up:f")
                    f ["v"] = (-radius, -radius, height, 128 - radius, -radius, height, -radius, 128 - radius, height)
                    f ["tex"] = tex
                    p1.appenditem (f)

                    f = quarkx.newobj ("down:f")
                    f ["v"] = (-radius, -radius, -height, -radius, 128 - radius, -height, 128 - radius, -radius, -height)
                    f ["m"] = "1"
                    f ["tex"] = tex
                    p1.appenditem (f)

                # We want walls...
                if walls:
                    v1 = self.ComputePoint (angle, radius - wallthickness, gridsize, snapgrid, -height)
                    v2 = self.ComputePoint (angle - step, radius - wallthickness, gridsize, snapgrid, -height)
                    v3 = self.ComputePoint (angle, (radius - wallthickness) - uprindent, gridsize, snapgrid, height)

                    f = quarkx.newobj ("side:f")
                    f ["v"] = (v1.x, v1.y, v1.z, v2.x, v2.y, v2.z, v3.x, v3.y, v3.z)
                    f ["tex"] = tex
                    p1.appenditem(f)

                p.appenditem(p1);

            # Next point
            angle = angle - step
            i = i + 1

        # Drop the items
        quarkpy.mapbtns.dropitemsnow (self.editor, [p], "make n sided prism")

        # commit the undo action
        #undo.ok (self.editor.Root, "make n sided prism")
        self.close()
        return

    # Compute the vertex
    def ComputePoint (self, angle, radius, gridsize, snapgrid, z):
        x = math.cos (angle) * radius
        y = math.sin (angle) * radius
        if snapgrid:
            x = quarkx.rnd (x / gridsize) * gridsize
            y = quarkx.rnd (y / gridsize) * gridsize
            z = quarkx.rnd (z / gridsize) * gridsize
        return quarkx.vect (x, y, z)

# Function to start the dialog
def MakePrismClick (m):
    editor = mapeditor ()
    if editor is None: return
    MakePrismDlg (quarkx.clickform, editor)

# Register the replace texture menu item
quarkpy.mapcommands.items.append(quarkpy.qmenu.sep)   # separator
quarkpy.mapcommands.items.append(quarkpy.qmenu.item("&Make Prism", MakePrismClick))

# ----------- REVISION HISTORY ------------
#
#
# $Log$
#
#
#


#!/usr/bin/env python
# -*-coding:utf-8 -*-
# created by 'zjh' on 18-1-26

from shapely.geometry import mapping, Polygon
import fiona

def geojson2shp(geojson, shpdst, id):
    '''

    :param geojson: the geojson format of a polygon
    :param shpdst: the path of the shapefile
    :param id: the id property
    :return: no return, just save the shapefile into the shpdst
    '''
    # an example Shapely geometry
    coordinates = geojson['coordinates']
    poly = Polygon(coordinates[0])

    # Define a polygon feature geometry with one attribute
    schema = {
        'geometry': 'Polygon',
        'properties': {'id': 'int'},
    }

    # Write a new Shapefile
    with fiona.open(shpdst, 'w', 'ESRI Shapefile', schema) as c:
        ## If there are multiple geometries, put the "for" loop here
        c.write({
            'geometry': mapping(poly),
            'properties': {'id': id},
        })


from nutils import *
from . import register, unittest

@register
def hierarchical():

  ref0, geom = mesh.rectilinear( [[0,1,2]] )
  e1, e2 = ref0
  ref1 = ref0.refined_by( [e2] )
  e3, e4, e5 = ref1
  ref2 = ref1.refined_by( [e4] )

  # Topologies:
  # ref0    [  .  .  .  |  .  .  .  ]
  # ref1    [  .  .  .  |  .  |  .  ]
  # ref2    [  .  .  .  |  |  |  .  ]
  # trimmed [  .  .  .  |]

  @unittest
  def untrimmed( makeplots=False ):
    basis = ref2.basis( 'std', degree=1 )
    assert basis.shape == (5,)
    x, y = ref2.elem_eval( [ geom[0], basis ], ischeme='bezier2', separate=False )
    assert numpy.all( y == .25 * numpy.array(
      [[4,0,0,0,0],
       [0,4,0,0,0],
       [0,3,2,0,4],
       [0,2,4,0,0],
       [0,0,0,4,0]] )[[0,1,1,2,2,3,3,4]] )
    if makeplots:
      with plot.PyPlot( 'basis' ) as plt:
        plt.plot( x, y )

  @unittest
  def trimmed( makeplots=False ):
    levelset = 1.125 - geom[0]
    trimmed = ref0.trim( levelset, maxrefine=3 ).refined_by( [e2] ).refined_by( [e4] )
    trimbasis = trimmed.basis( 'std', degree=1 )
    x, y = trimmed.simplex.elem_eval( [ geom[0], trimbasis ], ischeme='bezier2', separate=False )
    assert numpy.all( y == .125 * numpy.array(
      [[8,0,0],
       [0,8,0],
       [0,7,4]] )[[0,1,1,2]] )
    if makeplots:
      with plot.PyPlot( 'basis' ) as plt:
        plt.plot( x, y )

@register
def hierarchicalboundary():

  domain, geom = mesh.rectilinear( [[0,1,2],[0,1,2]] )
  left = domain[:1].withboundary( leftbnd=... )
  leftbasis = left.basis( 'std', degree=1 )
  right = domain[1:].withboundary( rightbnd=... )
  rightbasis = right.basis( 'std', degree=1 )
  trimmed = domain - right

  @unittest
  def volume():
    assert trimmed.boundary.integrate( 1, geometry=geom, ischeme='gauss1' ) == 6

  @unittest
  def boundary():
    assert trimmed.boundary['rightbnd'].integrate( 1, geometry=geom, ischeme='gauss1' ) == 2

  @unittest
  def left_boundary():
    left = trimmed.boundary['rightbnd']
    assert numpy.any( left.elem_eval( leftbasis, ischeme='gauss1', separate=False ) )
    assert not numpy.any( left.elem_eval( function.opposite(leftbasis), ischeme='gauss1', separate=False ) )
    assert numpy.any( left.elem_eval( function.opposite(rightbasis), ischeme='gauss1', separate=False ) )
    assert not numpy.any( left.elem_eval( rightbasis, ischeme='gauss1', separate=False ) )

@register
def specialcases():

  domain, geom = mesh.rectilinear( [[0,.5,1]]*2 )
  x, y = geom

  @unittest
  def almost_all_positive():
    domain.trim( (x-y) * (x-y+.25), maxrefine=1 )

  eps = .0001

  for xi, eta, direction in (x,y,'x'), (y,x,'y'):
    for perturb, how in (0,'mid'), (1,'pos'), (-1,'neg'), (xi-.5,'ramp'), (xi-eta,'tilt'):
      for maxrefine in 0, 1, 2:

        @unittest( name=direction+how+str(maxrefine) )
        def inter_elem_2d():
          domain.trim( eta-.75+eps*perturb, maxrefine=maxrefine )

      for maxrefine in 0, 1:

        @unittest( name=('x' if xi is x else 'y')+how+str(maxrefine) )
        def intra_elem_2d():
          pos = domain.trim( eta-.5+eps*perturb, maxrefine=maxrefine )
          pos.volume_check( geom )
          neg = domain - pos
          neg.volume_check( geom )

  domain, geom = mesh.rectilinear( [[0,.5],[0,.5],[0,.5,1]] )
  x, y, z = geom

  for perturb, how in (0,'mid'), (1,'pos'), (-1,'neg'), (x-.5,'ramp'), (x-y,'tilt'):
    for maxrefine in 0, 1, 2:

      @unittest( name=how+str(maxrefine) )
      def inter_elem_3d():
        domain.trim( z-.75+eps*perturb, maxrefine=maxrefine )

@register
def setoperations():

  domain, geom = mesh.rectilinear( [[-.5,-1./6,1./6,.5]]*2 ) # unit square
  x, y = geom
  bottomright = domain.trim( x-y, maxrefine=0, name='trim1' )
  right = bottomright.trim( x+y, maxrefine=0, name='trim2' )
  bottom = bottomright - right
  topleft = domain - bottomright
  top = topleft.trim( x+y, maxrefine=0, name='trim2' )
  left = topleft - top

  Lexact = 1+numpy.sqrt(2)
  for name, dom in ('left',left), ('top',top), ('right',right), ('bottom',bottom):
    @unittest( name=name )
    def boundary():
      L = dom.boundary.integrate( 1, geometry=geom, ischeme='gauss1' )
      assert numpy.isclose( L, 1+numpy.sqrt(2)  ), 'full boundary: wrong length: {} != {}'.format( L, 1+numpy.sqrt(2) )
      L = dom.boundary[name].integrate( 1, geometry=geom, ischeme='gauss1' )
      assert numpy.isclose( L, 1  ), '{}: wrong length: {} != {}'.format( name, L, 1 )
      L = dom.boundary['trim1' if name not in ('left','top') else 'trim1'].integrate( 1, geometry=geom, ischeme='gauss1' )
      assert numpy.isclose( L, .5*numpy.sqrt(2)  ), 'trim1: wrong length: {} != {}'.format( L, .5*numpy.sqrt(2) )
      L = dom.boundary['trim2' if name not in ('left','bottom') else 'trim2'].integrate( 1, geometry=geom, ischeme='gauss1' )
      assert numpy.isclose( L, .5*numpy.sqrt(2)  ), 'trim2: wrong length: {} != {}'.format( L, .5*numpy.sqrt(2) )

  @unittest
  def union():
    assert (top|left) | (right|bottom) == domain
    union = (right|left) | (top|bottom)
    assert isinstance( union, topology.UnionTopology )
    assert set(union) == set(domain)

@register( 'sphere', 3, 2, 3, 6e-3 )
@register( 'circle', 2, 2, 5, 2.1e-4 )
def cutdomain( ndims, nelems, maxrefine, errtol ):

  domain, geom = mesh.rectilinear( (numpy.linspace(0,1,nelems+1),)*ndims )
  radius = numpy.sqrt( .5 )
  levelset = radius**2 - ( geom**2 ).sum(-1)
  pos = domain.trim( levelset=levelset, maxrefine=maxrefine )
  neg = domain - pos
  V = 1.
  Vprev = 1. / (numpy.pi*radius)
  for idim in range( ndims ):
    S = Vprev * (2*numpy.pi*radius)
    Vprev = V
    V = S * (radius/(idim+1))
  exact_volume = V / 2**ndims
  exact_trimsurface = S / 2**ndims
  exact_totalsurface = exact_trimsurface + Vprev / (2**(ndims-1)) * ndims
  errtol = errtol

  @unittest
  def volume():
    volume = pos.volume( geom )
    volerr = abs( volume - exact_volume ) / exact_volume
    log.user( 'volume error:', volerr )
    assert volerr < errtol, 'volume tolerance not met: {:.2e} > {:.2e}'.format( volerr, errtol )

  for name, dom in ('pos',pos), ('neg',neg):
    @unittest( name=name )
    def div():
      dom.volume_check( geom, decimal=14 )

  @unittest
  def surface():
    trimsurface = pos.boundary['trimmed'].volume( geom )
    trimerr = abs( trimsurface - exact_trimsurface ) / exact_trimsurface
    log.user( 'trim surface error:', trimerr )
    totalsurface = pos.boundary.volume( geom )
    totalerr = abs( totalsurface - exact_totalsurface ) / exact_totalsurface
    log.user( 'total surface error:', totalerr )
    assert trimerr < errtol, 'trim surface tolerance not met: {:.2e} > {:.2e}'.format( trimerr, errtol )
    assert totalerr < errtol, 'total surface tolerance not met: {:.2e} > {:.2e}'.format( totalerr, errtol )

  @unittest
  def locate():
    curvegeom = geom * ( 1 + .1 * function.sin(function.norm2(geom)*numpy.pi/radius) ) # interface preserving non-polynomial scaling
    for p in numpy.linspace( .001, .999, 20 ):
      point = p * .5**numpy.arange(domain.ndims)
      r = numpy.linalg.norm( point )
      try:
        ptopo = pos.locate( curvegeom, points=[point] )
      except topology.LocateError:
        assert r > radius
      else:
        assert r <= radius
        x, = ptopo.elem_eval( curvegeom, 'gauss1' )
        numpy.testing.assert_almost_equal( x, point )

@register
def multitrim():
  domain, geom = mesh.rectilinear( [[-1,1],[-1,1]] )
  geom_rel = ( function.rotmat(numpy.pi/6) * geom ).sum(-1)
  for itrim in range(4):
    domain = domain.trim( .7+(1-itrim%2*2)*geom_rel[itrim//2], maxrefine=1, name='trim{}'.format(itrim), ndivisions=16 )
  for itrim in range(4):
    L = domain.boundary['trim{}'.format(itrim)].integrate( 1, geometry=geom, ischeme='gauss1' )
    numpy.testing.assert_almost_equal( L, 1.4, decimal=4 )
  L = domain.boundary.integrate( 1, geometry=geom, ischeme='gauss1' )
  numpy.testing.assert_almost_equal( L, 5.6, decimal=4 )

@register
def leveltopo():
  domain0, geom = mesh.rectilinear( [2,2] )
  domain1 = domain0.refined

  @unittest
  def uniform():
    domain2 = domain1.refined
    basis = domain0.basis( 'std', degree=1 )
    level = basis.dot( (numpy.arange(len(basis))%2)-.5 )
    trimtopoA = domain0.trim( level, maxrefine=2 )
    trimtopoB = domain0.trim( level, maxrefine=2, leveltopo=domain2 )
    assert trimtopoA.elements == trimtopoB.elements

  @unittest( raises=Exception )
  def uniformfail():
    domain2 = domain1.refined
    basis = domain0.basis( 'std', degree=1 )
    level = basis.dot( (numpy.arange(len(basis))%2)-.5 )
    trimtopo = domain0.trim( level, maxrefine=1, leveltopo=domain2 )

  @unittest
  def hierarchical():
    domain2 = domain1.refined_by( domain1.elements[:1] )
    basis = domain2.basis( 'std', degree=1 )
    level = basis.dot( (numpy.arange(len(basis))%2)-.5 )
    trimtopo = domain0.trim( level, maxrefine=2, leveltopo=domain2 )

  @unittest( raises=Exception )
  def hierarchicalfail():
    domain2 = domain1.refined_by( domain1.elements[:1] )
    basis = domain2.basis( 'std', degree=1 )
    level = basis.dot( (numpy.arange(len(basis))%2)-.5 )
    trimtopo = domain0.trim( level, maxrefine=1, leveltopo=domain2 )

@register
def trim_conforming():

  domain, geom = mesh.rectilinear( [4,4] )

  @unittest
  def untrimmed():
    assert len(domain.interfaces) == 24
    assert len(domain.boundary) == 16

  domain1 = domain.trim( 3-geom[0], maxrefine=2, name='trimright' )

  @unittest
  def trimright():
    assert len(domain1.interfaces) == 17
    assert len(domain1.boundary) == 14
    assert len(domain1.boundary['trimright']) == 4

  domain2 = domain1.trim( 3-geom[1], maxrefine=2, name='trimtop' )

  @unittest
  def trimtop():
    assert len(domain2.interfaces) == 12
    assert len(domain2.boundary) == 12
    assert len(domain2.boundary['trimright']) == 3
    assert len(domain2.boundary['trimtop']) == 3

  domain3 = domain2.trim( geom[0]-1, maxrefine=2, name='trimleft' )

  @unittest
  def trimleft():
    assert len(domain3.interfaces) == 7
    assert len(domain3.boundary) == 10
    assert len(domain3.boundary['trimright']) == 3
    assert len(domain3.boundary['trimtop']) == 2
    assert len(domain3.boundary['trimleft']) == 3

  domain4 = domain3.trim( geom[1]-1, maxrefine=2, name='trimbottom' )

  @unittest
  def trimbottom():
    assert len(domain4.interfaces) == 4
    assert len(domain4.boundary) == 8
    assert len(domain4.boundary['trimright']) == 2
    assert len(domain4.boundary['trimtop']) == 2
    assert len(domain4.boundary['trimleft']) == 2
    assert len(domain4.boundary['trimbottom']) == 2

  domain5 = domain.trim( 3-geom[0], maxrefine=2, name='trimtopright' ).trim( 3-geom[1], maxrefine=2, name='trimtopright' )

  @unittest
  def trimtopright():
    assert len(domain5.interfaces) == 12
    assert len(domain5.boundary) == 12
    assert len(domain5.boundary['trimtopright']) == 6


def match_tickers(tickers):
    return [U[U.ticker.str.startswith(x.upper())].ticker.iloc[0]
            for x in tickers]

def save_basket(text):
    baskets = get_all_baskets()
    baskets[bname] = {'name': text.split(' ')[0],
                      'desc': '',
                      'tickers': match_tickers(text.split(' ')[1:])}
    save_all_baskets(baskets)
    return 'Basket '+bname+' saved'

# basket = dict{ name, desc, tickers[] }
def get_all_baskets():
    b = pickle.load(open(BASKETS_PATH, 'rb'))
    return b

def save_all_baskets(baskets):
    with open(BASKETS_PATH, 'wb') as tmp:
        pickle.dump(baskets, tmp, protocol=pickle.HIGHEST_PROTOCOL)
    return 'Baskets saved'

def print_baskets(baskets):
    return '\n'.join([x + ':\n' +
                      (b[x]['desc'] + '\n' if b[x]['desc'] != '' else '') +
                      '* ' + '\n* '.join(b[x]['tickers']) + '\n'
                      for x in baskets])

def delete_basket(text):
    baskets = get_all_baskets()
    del baskets[text.split(' ')[0]]
    save_all_baskets(baskets)
    return 'Basket ' + bname + ' deleted'

def my_who(text):
    res = U[U.ticker.str.startswith(text.upper())]
    if len(res) > 0:
        return '\n'.join(res.apply(lambda x: str(x.CODE) + '\n(' +
                                   str(x.NAME) + ')\nmcap: ' +
                                   str(x.MCAP) + '\n', axis=1))
    else:
        return 'No matching securities'


from collections import Counter
import string
import re
import tensorflow as tf
import sys
from load_dataset import get_preprocessed_dataset, load_json_file
from model import my_QAmodel
from main import set_hyperparameter
def copy_normalize_answer(s):
    """Lower text and remove punctuation, articles and extra whitespace."""
    def remove_articles(text):
        return re.sub(r'\b(a|an|the)\b', ' ', text)

    def white_space_fix(text):
        return ' '.join(text.split())

    def remove_punc(text):
        exclude = set(string.punctuation)
        return ''.join(ch for ch in text if ch not in exclude)

    def lower(text):
        return text.lower()

    return white_space_fix(remove_articles(remove_punc(lower(s))))

def copy_f1_score(prediction, ground_truth):
    prediction_tokens = copy_normalize_answer(prediction).split()
    ground_truth_tokens = copy_normalize_answer(ground_truth).split()
    common = Counter(prediction_tokens) & Counter(ground_truth_tokens)
    num_same = sum(common.values())
    if num_same == 0:
        return 0
    precision = 1.0 * num_same / len(prediction_tokens)
    recall = 1.0 * num_same / len(ground_truth_tokens)
    f1 = (2 * precision * recall) / (precision + recall)
    return f1

def copy_exact_match_score(prediction, ground_truth):
    return (copy_normalize_answer(prediction) == copy_normalize_answer(ground_truth))

def copy_metric_max_over_ground_truths(metric_fn, prediction, ground_truths):
    scores_for_ground_truths = []
    for ground_truth in ground_truths:
        score = metric_fn(prediction, ground_truth)
        scores_for_ground_truths.append(score)
    return max(scores_for_ground_truths)

def copy_evaluate(dataset, predictions):
    f1 = exact_match = total = 0
    for article in dataset:
        for paragraph in article['paragraphs']:
            for qa in paragraph['qas']:
                total += 1
                if qa['id'] not in predictions:
                    message = 'Unanswered question ' + qa['id'] + \
                              ' will receive score 0.'
                    print(message, file=sys.stderr)
                    continue
                ground_truths = list(map(lambda x: x['text'], qa['answers']))
                prediction = predictions[qa['id']]
                exact_match += copy_metric_max_over_ground_truths(
                    copy_exact_match_score, prediction, ground_truths)
                f1 += copy_metric_max_over_ground_truths(
                    copy_f1_score, prediction, ground_truths)

    exact_match = 100.0 * exact_match / total
    f1 = 100.0 * f1 / total

    return {'exact_match': exact_match, 'f1': f1}

if __name__=='__main__':
    ##################################################################################
    # Setting: your path and file directory configuration
    mode = 'dev'
    data_path = 'D:/Wisdom/git/tmp/BIDAF/data/squad/'
    version = 'small'  # 'v1.1'
    save_path = "./saved_model/"
    meta_model_file = save_path + 'my_test_model-{2019-07-18-14-23}.ckpt.meta'
    whole_model_file = save_path + 'my_test_model-{2019-07-18-14-23}.ckpt'
    ##################################################################################
    # Dataset loading part
    json_file = data_path + mode + '-' + version + '.json'
    print('Start dataset loading')
    print('File:' + json_file)
    data = load_json_file(json_file)
    config, full_dataset, sub_info_dataset, index_dataset, word2vec_dataset = \
        get_preprocessed_dataset(data, mode=mode)
    print('Finish dataset loading')
    ##################################################################################
    # Setting: Tensorflow session
    config_tf = tf.ConfigProto()
    config_tf.log_device_placement = True
    config_tf.gpu_options.allow_growth = True
    ##################################################################################
    d, batch_size, char_vec_dim, is_training, config = set_hyperparameter(config)
    config['is_training'] = False

    QAmodel = my_QAmodel(config=config)
    config_tf = tf.ConfigProto()
    config_tf.log_device_placement = True
    config_tf.gpu_options.allow_growth = True
    sess = tf.Session(config=config_tf)
    loader = tf.train.import_meta_graph(meta_model_file)
    loader.restore(sess, whole_model_file)

    # graph = tf.get_default_graph()
    # print([n.name for n in tf.get_default_graph().as_graph_def().node])
    # w1 = graph.get_tensor_by_name("w1:0")
    # p1=graph.get_tensor_by_name("p1:0")
    # p2 = graph.get_tensor_by_name("p2:0")


"""
This module holds the RequestHandler class and all of its functionalities, as well as a custom exception class,
BadRequestError, and an Enum, PokedexModes. It also contains helper static methods.
"""

import asyncio
import os
from pathlib import Path

import aiohttp

from pokeretriever.pokedex_object_factory import get_pokedex_factory

class BadRequestError(Exception):
    """
    Custom class that has the purpose of being thrown when there is a problem in the request being made.
    """

    def __init__(self):
        """
        Initialize a new BadRequestError instance and pass in a message with details of this error to the parent class,
        Exception.
        """
        super().__init__("An error occurred. Skipping this request.\n")

class RequestHandler:
    """
    Class that is responsible for making the requests to a specific API. In this program, we use the pokemon api.
    """

    def __init__(self, request):
        """
        Instantiate a RequestHandler with a request.
        :param request: a Request
        """
        self._request = request

    @property
    def request(self):
        return self._request

    async def _get_pokedex_object(self, request_input: str, url: str, session: aiohttp.ClientSession):
        """
        Get the Pokedex object from the API via a GET request.
        :param request_input: name or id (a String or an int)
        :param url: a String
        :param session: a ClientSession
        :return: a PokedexObject
        """

        target_url = url.format(self.request.mode, request_input)

        response = await session.request(method="GET", url=target_url)

        if response.status == 404:
            raise BadRequestError()

        json_dict = await response.json()

        # Get factory for mode passed in
        factory = get_pokedex_factory(self.request.mode)

        return RequestHandler.make_pokedex_object(self.request.expanded, factory, **json_dict)

    @staticmethod
    def make_pokedex_object(expanded, factory, **kwargs):
        """
        Make and return a new PokedexObject and personalize it according to the expanded flag, the factory of the
        PokedexObject and the dictionary that contains the attributes, kwargs.
        :param expanded: a bool
        :param factory: a PokedexFactory
        :param kwargs: a dict
        :return: a PokedexObject
        """
        pokedex_object = factory.create_pokedex_object(expanded, **kwargs)
        return pokedex_object

    async def process_requests(self):
        """
        Read the request input and start the chain to process the request.
        Quit the program if no input data or file is found.
        :return:
        """
        inputs = []
        if self.request.input_data:
            inputs.append(self.request.input_data)
        elif self.request.input_file:
            inputs = self._get_input_list()
        else:
            print("No input data or input file found. Please try again.")
            quit()

        url = "https://pokeapi.co/api/v2/{}/{}/"
        async with aiohttp.ClientSession() as session:
            print("Getting pokedex object...\n")

            # process each request
            async_coroutines = [self._get_pokedex_object(request_input, url, session)
                                for request_input in inputs]

            responses = await asyncio.gather(*async_coroutines, return_exceptions=True)
            return responses

    def _get_input_list(self) -> list:
        """
        Get lines of input from an input file as a list.
        :return: a list
        """
        request_input = []
        filename = self.request.input_file
        extension = os.path.splitext(filename)[1]
        if RequestHandler.check_inputfile(filename, extension):
            with open(self.request.input_file, mode='r') as data_file:
                data = data_file.readlines()
            for i in data:
                request_input.append(i.rstrip("\n"))
        else:
            print("Invalid file. Please enter a existing text file.")
            quit()
        return request_input

    @staticmethod
    def check_inputfile(filepath, file_extension):
        """
        Check the if the filepath exists and if the provided file is a text file.
        :param filepath: a String
        :param file_extension: a String
        :return: True if the file is valid, False if it is not
        """
        if Path(filepath).exists():
            if file_extension == ".txt":
                return True
        return False


import multiprocessing,os

def f(conn):
    conn.send([42,None,"hello from child_coon"])
    conn.send("childpid%s"%os.getpid())
    print("from parent:",conn.recv())

if __name__ == "__main__":
    parent_conn,child_conn = multiprocessing.Pipe()
    p = multiprocessing.Process(target = f,args = (child_conn,))
    p.start()
    print(parent_conn.recv())
    print(parent_conn.recv())
    parent_conn.send("from parent")
    p.join()


# -*- coding: utf-8 -*-

from berserker_resolver.compat import get_version, xrange_compat
from berserker_resolver.base import set_kwargs

_version = get_version()

if _version == 3:
    from queue import Queue, Empty
elif _version == 2:
    from Queue import Queue, Empty

from threading import Thread
from math import ceil

class ConcurrenceBase(object):
    '''
    Base class for concurrence features.
    '''
    threads = 1024

    def __init__(self, **kwargs):
        kwargs = set_kwargs(self, kwargs, ['threads'])
        super(ConcurrenceBase, self).__init__(**kwargs)

    def chunk_domains(self, domains):
        '''
        Chunks domains list to small lists for each thread.
        '''
        domains_one_thread = int(ceil(float(len(domains))/self.threads))
        return self._chunk_list(domains, domains_one_thread)

    def _chunk_list(self, l, n):
        for i in xrange_compat(0, len(l), n):
            yield l[i:i+n]

class ThreadConcurrence(ConcurrenceBase):
    queue = Queue()

    def __init__(self, **kwargs):
        super(ThreadConcurrence, self).__init__(**kwargs)

    def thread_resolve(self, domains, **query_args):
        chunks = self.chunk_domains(domains)
        threads = []
        for i in xrange_compat(self.threads):
            try:
                chunk = next(chunks)
            except StopIteration:
                break
            t = Thread(target=self.__thread, args=(chunk, query_args))
            t.start()
            threads.append(t)
        [t.join() for t in threads]
        result = []
        while True:
            try:
                result.append(self.queue.get(block=False))
            except Empty:
                break
        return result

    def __thread(self, domains, query_args):
        for d in domains:
            self.queue.put(self.query(d, **query_args))

    def query(self, domain, **query_args):
        return NotImplementedError('Function for query one domain, implement it.')


import argparse
import collections
import json
import os
import sys

from vocab import Dictionary
from eval_util import *
from dstore_util import *
from dataset_util import *
from ngram_util import *

import numpy as np
import torch

from tqdm import tqdm

def main(args):
    use_cuda = True
    device = torch.cuda.current_device() if use_cuda else None
    dkey = 'approx_dist' if args.approx else 'dist'

    context = DatasetUtils().build_context(args, include_keys=False)

    p = context['test']['p']
    dist = context['test'][dkey]
    tgts = context['test']['tgts']
    src = make_src(tgts, args.max_ngram)
    knn_tgts = context['test']['knn_tgts']
    knns = context['test']['knns']

    knn_dstore = Dstore(args.knn_dstore, args.knn_dstore_size, 1024)
    knn_dstore.initialize(include_keys=False)

    trn_tgt = npy_copy(knn_dstore.tgts[:])
    trn_src = make_src(trn_tgt, args.max_ngram)
    ngram_lm = build_ngram_lm(trn_src, trn_tgt, limit=args.ngram_limit, max_ngram=args.max_ngram)

    print('read vocab')
    vocab = Dictionary()
    vocab.add_from_file(args.vocab)
    vocab.finalize()
    vocab_size = len(vocab)
    print('found {} tokens'.format(len(vocab)))
    print('')

    def run_eval(context):
        coeff = context['coeff']
        knns = context['knns']
        knn_tgts = context['knn_tgts']
        src = context['src']
        tgts = context['tgts']
        dist = context['dist']
        p = context['p']

        label = (knn_tgts == tgts.reshape(-1, 1, 1)).astype(np.int)

        knn_p = EvalUtil.get_knn_log_prob(tgts, dist, knn_tgts)

        ngram_p = ngram_lm.batch_predict(src, tgts)
        ngram_logp = np.log(np.clip(ngram_p, 1e-8, 1))
        ngram_logp[ngram_p == 0] = -10000
        del ngram_p

        p_ = torch.from_numpy(p).float()
        knn_p_ = torch.from_numpy(knn_p).float()
        ngram_p_ = torch.from_numpy(ngram_logp).float()

        unif_p_ = p_.clone()
        unif_p_[:] = np.log(1/vocab_size)

        #
        unif_knn_p = EvalUtil.combine_knn_and_vocab_probs(
                    knn_p_,
                    unif_p_,
                    0.9)
        unif_ngram_p = EvalUtil.combine_knn_and_vocab_probs(
                    ngram_p_,
                    unif_p_,
                    0.9)

        interp_knn_p = EvalUtil.combine_knn_and_vocab_probs(
                    knn_p_,
                    p_,
                    coeff)
        interp_ngram_p = EvalUtil.combine_knn_and_vocab_probs(
                    ngram_p_,
                    p_,
                    coeff)

        interp_many_p = EvalUtil.combine_many_probs(
                [knn_p_, ngram_p_],
                p_,
                [0.1, 0.2]
                )

        ppl = EvalUtil.eval_ppl(p)
        knn_ppl = EvalUtil.eval_ppl(unif_knn_p)
        ngram_ppl = EvalUtil.eval_ppl(unif_ngram_p)
        interp_knn_ppl = EvalUtil.eval_ppl(interp_knn_p)
        interp_ngram_ppl = EvalUtil.eval_ppl(interp_ngram_p)
        interp_many_ppl = EvalUtil.eval_ppl(interp_many_p)

        n = tgts.shape[0]

        print('n = {}, coeff = {:.3f}, ppl = {:.3f}, knn_ppl = {:.3f}, ngram_ppl = {:.3f}, i_knn_ppl = {:.3f}, i_ngram_ppl = {:.3f}, i_many_ppl = {:.3f}'.format(
            n, coeff, ppl, knn_ppl, ngram_ppl, interp_knn_ppl, interp_ngram_ppl, interp_many_ppl))

    def print_header(x, l=8, n=40):
        line = '-' * l
        line += ' ' + x + ' '
        line += '-' * (n - len(line))
        print(line)

    # COMPUTE ALL EVAL
    print_header('EVAL ALL')
    context = {}
    context['coeff'] = 0.25
    context['knns'] = knns
    context['knn_tgts'] = knn_tgts
    context['src'] = src
    context['tgts'] = tgts
    context['dist'] = dist
    context['p'] = p
    run_eval(context)
    print('')

    print_header('many coeff')
    coeff_list = np.arange(1, 10) / 20
    for coeff in coeff_list:
        context['coeff'] = coeff
        context['knns'] = knns
        context['knn_tgts'] = knn_tgts
        context['src'] = src
        context['tgts'] = tgts
        context['dist'] = dist
        context['p'] = p
        run_eval(context)
    print('')

    # COMPUTE FILTERED EVAL

    w_counts = np.array(vocab.count).reshape(-1)
    src_counts = w_counts[src[:, -1].reshape(-1)]
    tgt_counts = w_counts[tgts.reshape(-1)]

    t_list = [10**i for i in range(1, 7)]

    print('\n' + 't is src')

    for i in range(0, len(t_list)):
        t0 = t_list[i]
        t1 = np.inf if (i+1) >= len(t_list) else t_list[i+1]

        mask = np.logical_and(src_counts >= t0, src_counts < t1)

        print_header('interval = {}:{}'.format(t0, t1))

        context = {}
        context['coeff'] = 0.25
        context['knns'] = knns[mask]
        context['knn_tgts'] = knn_tgts[mask]
        context['src'] = src[mask]
        context['tgts'] = tgts[mask]
        context['dist'] = dist[mask]
        context['p'] = p[mask]
        run_eval(context)

        print('')

    print('\n' + 't is tgt')

    for i in range(0, len(t_list)):
        t0 = t_list[i]
        t1 = np.inf if (i+1) >= len(t_list) else t_list[i+1]

        mask = np.logical_and(tgt_counts >= t0, tgt_counts < t1)

        print_header('interval = {}:{}'.format(t0, t1))

        context = {}
        context['coeff'] = 0.25
        context['knns'] = knns[mask]
        context['knn_tgts'] = knn_tgts[mask]
        context['src'] = src[mask]
        context['tgts'] = tgts[mask]
        context['dist'] = dist[mask]
        context['p'] = p[mask]
        run_eval(context)

        print('')

    print('\n' + 'use both')

    for i in range(0, len(t_list)):
        for j in range(0, len(t_list)):
            t0_src = t_list[i]
            t1_src = np.inf if (i+1) >= len(t_list) else t_list[i+1]

            t0_tgt = t_list[j]
            t1_tgt = np.inf if (j+1) >= len(t_list) else t_list[j+1]

            mask_src = np.logical_and(src_counts >= t0_src, src_counts < t1_src)
            mask_tgt = np.logical_and(tgt_counts >= t0_tgt, tgt_counts < t1_tgt)
            mask = np.logical_and(mask_src, mask_tgt)

            print_header('interval = src:{}:{} tgt:{}:{}'.format(t0_src, t1_src, t0_tgt, t1_tgt))

            context = {}
            context['coeff'] = 0.25
            context['knns'] = knns[mask]
            context['knn_tgts'] = knn_tgts[mask]
            context['src'] = src[mask]
            context['tgts'] = tgts[mask]
            context['dist'] = dist[mask]
            context['p'] = p[mask]
            run_eval(context)

            print('')

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    # dstore
    parser.add_argument('--dstore', default='dstore_valid', type=str)
    parser.add_argument('--dstore-size', default=217646, type=int)
    parser.add_argument('--vocab', default='data-bin/wikitext-103/dict.txt')
    #
    parser.add_argument('--test-dstore', default='dstore_test', type=str)
    parser.add_argument('--test-dstore-size', default=245569, type=int)
    parser.add_argument('--test-lookup', default='dstore_test/lookup')
    # dstore
    parser.add_argument('--knn-dstore', default='dstore_train', type=str)
    parser.add_argument('--knn-dstore-size', default=103225485, type=int)
    # dstore neighbors
    parser.add_argument('--lookup', default='dstore_valid/lookup', type=str)
    parser.add_argument('--lookup-k', default=1024, type=int)
    # dstore
    parser.add_argument('--vocab_threshold', default=1000, type=int)

    # examine
    parser.add_argument('--k', default=1024, type=int)
    # debug
    parser.add_argument('--limit', default=-1, type=int)
    parser.add_argument('--ngram-limit', default=-1, type=int, help='Number of training examples for ngram lm.')
    parser.add_argument('--max-ngram', default=2, type=int, help='Token width of ngram lm.')
    parser.add_argument('--preset', default='valid', type=str)
    parser.add_argument('--approx', action='store_true')
    args = parser.parse_args()

    # Print flags.
    print(args)

    main(args)