# -*- coding: utf-8 -*-
"""
Created on Wed Aug 11 14:46:09 2021

@author: Olivia Gozel

Get the pairwise correlation as a function of distance data for Peijia
"""

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import os

os.chdir('C:\\Users\\olivi\\Dropbox\\Projects\\U19_project\\Code_python\\')

import globalParams
import functions_analyze

dataDir = 'C:\\Users\\olivi\\Dropbox\\Projects\\U19_project\\U19data\\'

peijiaDir = 'C:\\Users\\olivi\\Dropbox\\Projects\\U19_project\\Peijia_project\\'

#%% Parameters of the preprocessed data to analyze

dataType = 'L23_thalamicBoutons' # 'L4_cytosolic' or 'L23_thalamicBoutons'
idxDataset = 0
# L4 good: 0,1,2,3,4,5,6,7,8,9
# L2/3: 2 is weird (very high pairwise correlation at long distances)

filepath = dataDir + dataType + '_dataSpecs.hdf'
dataSpecs = pd.read_hdf(filepath)

dataDate = dataSpecs.iloc[idxDataset]['Date']
dataMouse = dataSpecs.iloc[idxDataset]['Mouse']
dataDepth = dataSpecs.iloc[idxDataset]['Depth']
pixelSize = dataSpecs.iloc[idxDataset]['PixelSize']
dataSessions = dataSpecs.iloc[idxDataset]['Sessions']

# Choose a neuropil factor, by default: globalParams.neuropilSub[3]
# NB: dataType='L23_thalamicBoutons' with idxDataset=2 also has
#       globalParams.neuropilSub[4]
dataNeuropilSub = globalParams.neuropilSub[3]

#%% Load data

filepath = globalParams.processedDataDir + dataType +'_' + dataDate + '_' + \
        dataMouse + '_' + dataDepth + '_neuropilF_' + dataNeuropilSub + '.hdf'

fluo = pd.read_hdf(filepath,'fluo')
charROI = pd.read_hdf(filepath,'charROI')
charTrials = pd.read_hdf(filepath,'charTrials')
positionROI = pd.read_hdf(filepath,'positionROI')
distROI = pd.read_hdf(filepath,'distROI')

#%% Noise pairwise correlation as a function of pairwise distance
# Neuropil factor = 0.75
# 5um distance bins, first one is between 5um and 10um (centered in 7.5um)
# OS ROIs only
# Stimulus frames only

binCenters, sortedPairCorr = functions_analyze.plot_noiseCorr_fdist(
    fluo, distROI, charROI, charTrials, boolVisuallyEvoked=True, boolOS=True,
    boolIdenticalTuning=False, boolStimulus=True, startBin=5 ,binSize=5)

# Save it in excel
df = pd.DataFrame(sortedPairCorr)
#writer = pd.ExcelWriter(peijiaDir+'L23cytosolic_pairCorr_fDist.xlsx',
#                        engine='openpyxl') # when the file does not exist yet
writer = pd.ExcelWriter(peijiaDir+'L23cytosolic_pairCorr_fDist.xlsx',
                        engine='openpyxl', mode='a') # once the file exists
df.to_excel(writer, sheet_name='Dataset'+str(idxDataset))
writer.save()

#%% Noise pairwise correlation as a function of pairwise distance for similarly
# tuned ROIs
# Neuropil factor = 0.75
# 5um distance bins, first one is between 5um and 10um (centered in 7.5um)
# OS ROIs only, separately for each orientation selectivity
# Stimulus frames only
# For pairs of OS ROIs with same orientation tuning (DeltaTheta=0), combined
# for all four orientations

binCenters, sortedPairCorr = functions_analyze.plot_noiseCorr_fdist(
    fluo, distROI, charROI, charTrials, boolVisuallyEvoked=True, boolOS=True,
    boolIdenticalTuning=True, boolStimulus=True, startBin=5, binSize=5)

# Save it in excel
df = pd.DataFrame(sortedPairCorr)
# writer = pd.ExcelWriter(
#     peijiaDir+'L23cytosolic_pairCorr_fDist_sameTuning.xlsx',
#     engine='openpyxl') # when the file does not exist yet
# writer = pd.ExcelWriter(
#     peijiaDir+'L23cytosolic_pairCorr_fDist_sameTuning.xlsx',
#     engine='openpyxl', mode='a') # once the file exists
# df.to_excel(writer, sheet_name='Dataset'+str(idxDataset))
# writer.save()

#%% Select 5 ROIs

nFramesPerTrial = 30
nTrialsPerOri = 250
idxT135 = np.where(charTrials['Orientation']==135)[0]
ref_ROI = 87 # prefOri=135°
close_sameTuning_ROI = 90 # prefOri=135°, 8.7um apart
close_diffTuning_ROI = 14 # prefOri=270°, 19.7um apart
far_sameTuning_ROI = 82 #142 # prefOri=135°, 83.8um apart # 72.4um apart
far_diffTuning_ROI = 100 # prefOri=270°, 84.0um apart # 63.1um apart

# Trials of the preferred orientation of the reference ROI
thisFluo_ref = np.array(fluo[ref_ROI].loc[idxT135])
thisFluo_closeST = np.array(fluo[close_sameTuning_ROI].loc[idxT135])
thisFluo_closeDT = np.array(fluo[close_diffTuning_ROI].loc[idxT135])
thisFluo_farST = np.array(fluo[far_sameTuning_ROI].loc[idxT135])
thisFluo_farDT = np.array(fluo[far_diffTuning_ROI].loc[idxT135])

print('Correlation of ref with close sameTuning: ' +
      str(np.corrcoef(thisFluo_ref - np.mean(thisFluo_ref),
                      thisFluo_closeST- np.mean(thisFluo_closeST))[0,1])
      )
print('Correlation of ref with close diffTuning: ' +
      str(np.corrcoef(thisFluo_ref - np.mean(thisFluo_ref),
                      thisFluo_closeDT- np.mean(thisFluo_closeDT))[0,1])
      )
print('Correlation of ref with far sameTuning: ' +
      str(np.corrcoef(thisFluo_ref - np.mean(thisFluo_ref),
                      thisFluo_farST- np.mean(thisFluo_farST))[0,1])
      )
print('Correlation of ref with far diffTuning: ' +
      str(np.corrcoef(thisFluo_ref - np.mean(thisFluo_ref),
                      thisFluo_farDT- np.mean(thisFluo_farDT))[0,1])
      )

theseTrials = np.arange(0,100,1) #np.array((3,4,12,13,20))
for i in range(len(theseTrials)):
    thisT = theseTrials[i]
    plt.figure()
    plt.plot(thisFluo_ref[thisT*nFramesPerTrial:(thisT+1)*nFramesPerTrial]);
    plt.plot(thisFluo_closeST[thisT*nFramesPerTrial:(thisT+1)*nFramesPerTrial]);
    plt.plot(thisFluo_closeDT[thisT*nFramesPerTrial:(thisT+1)*nFramesPerTrial]);
    plt.plot(thisFluo_farST[thisT*nFramesPerTrial:(thisT+1)*nFramesPerTrial]);
    plt.plot(thisFluo_farDT[thisT*nFramesPerTrial:(thisT+1)*nFramesPerTrial]);
    plt.legend(['ref', 'close sameTuning', 'close diffTuning','far sameTuning',
                'far diffTuning'])
    plt.title('Trial=' + str(thisT));

# Plot individual fluorescence traces for the 5 ROIs of interest, for a
# selection of trials
theseTrials = np.array((3,22,30,39,62,63,82,85))
for i in range(len(theseTrials)):
    thisT = theseTrials[i]
    fig,axs = plt.subplots(1,5, constrained_layout=True, sharey=True)
    axs[0].plot(thisFluo_ref[thisT*nFramesPerTrial:(thisT+1)*nFramesPerTrial]);
    axs[0].set_title(str(ref_ROI))
    axs[1].plot(thisFluo_closeST[thisT*nFramesPerTrial:(thisT+1)*nFramesPerTrial]);
    axs[1].set_title(str(close_sameTuning_ROI))
    axs[2].plot(thisFluo_closeDT[thisT*nFramesPerTrial:(thisT+1)*nFramesPerTrial]);
    axs[2].set_title(str(close_diffTuning_ROI))
    axs[3].plot(thisFluo_farST[thisT*nFramesPerTrial:(thisT+1)*nFramesPerTrial]);
    axs[3].set_title(str(far_sameTuning_ROI))
    axs[4].plot(thisFluo_farDT[thisT*nFramesPerTrial:(thisT+1)*nFramesPerTrial]);
    axs[4].set_title(str(far_diffTuning_ROI))
    #fig.suptitle('Trial='+str(thisT));
    fig.savefig(peijiaDir + 'trial' + str(thisT) + '.eps', format='eps')

# Plot average fluorescence trace for each orientation for the ROIs of interest
thisROI = 100
data = pd.concat([charTrials[['TrialFrame','Orientation']], fluo[thisROI]],
                 axis=1)
avgPerOri = data.groupby(['TrialFrame','Orientation']).mean()
avgPerOri = avgPerOri.sort_values(['Orientation','TrialFrame'])
avgPerOri = avgPerOri.reset_index()
stdPerOri = data.groupby(['TrialFrame','Orientation']).std()
stdPerOri = stdPerOri.sort_values(['Orientation','TrialFrame'])
stdPerOri = stdPerOri.reset_index()

fig, axs = plt.subplots(2, 2, constrained_layout=True, sharey=True)
axs = axs.ravel()
for o in range(globalParams.nOri):
    tmpAvg = avgPerOri[avgPerOri['Orientation']==globalParams.ori[o]].drop(
        ['TrialFrame','Orientation'], axis=1)
    tmp_avg = np.squeeze(np.array(tmpAvg))
    tmpStd = stdPerOri[stdPerOri['Orientation']==globalParams.ori[o]].drop(
        ['TrialFrame','Orientation'], axis=1)
    tmp_std = np.squeeze(np.array(tmpStd))
    tmp_sem = tmp_std/np.sqrt(nTrialsPerOri)
    axs[o].fill_between(np.arange(0, nFramesPerTrial),
                        tmp_avg-tmp_sem, tmp_avg+tmp_sem)
    axs[o].set_title(str(globalParams.ori[o]) + '°')
fig.suptitle('ROI ' + str(thisROI))

# Plot the ROIs
import functions_postprocess
fluoPlaneWidthHeight = pd.read_hdf(filepath,'fluoPlaneWidthHeight')
fluoPlaneWidth = np.array(fluoPlaneWidthHeight)[0].item()
fluoPlaneHeight = np.array(fluoPlaneWidthHeight)[1].item()

# Hack to reuse function script (just need to change legend)
# cbar.ax.set_yticklabels(['90','14','82','100','87 (ref)','other'])
thisCharROI = charROI
thisCharROI['VisuallyEvoked'] = False
thisCharROI['VisuallyEvoked'].loc[ref_ROI] = True
thisCharROI['OS'].loc[ref_ROI] = False
thisCharROI['VisuallyEvoked'].loc[close_sameTuning_ROI] = True
thisCharROI['PrefOri'].loc[close_sameTuning_ROI] = 45
thisCharROI['VisuallyEvoked'].loc[close_diffTuning_ROI] = True
thisCharROI['PrefOri'].loc[close_diffTuning_ROI] = 135
thisCharROI['VisuallyEvoked'].loc[far_sameTuning_ROI] = True
thisCharROI['PrefOri'].loc[far_sameTuning_ROI] = 180
thisCharROI['VisuallyEvoked'].loc[far_diffTuning_ROI] = True
thisCharROI['PrefOri'].loc[far_diffTuning_ROI] = 270
functions_postprocess.plot_ROIwithSelectivity(thisCharROI, positionROI,
                                              fluoPlaneWidth, fluoPlaneHeight)


#####################################
# Developed by Baptiste PICARD      #
# picard.baptiste@laposte.net       #
# Started the 11th of April 2020    #
# picard.baptiste@laposte.net       #
#                                   #
#####################################

# Imports
# Librairies
import json
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt

# Modules
from account import *
from access import *
from utils import *
from maths import *

class Stock() :
	# Constructor
	def __init__(self, name, api_key, json, money) :
		self.stock_name = name
		self.api_key = api_key
		self.json = json
		self.money = money
		self.date = None
		self.day_ago = None
		self.week_ago = None
		self.month_ago = None
		self.df = None
		self.account = Account(name, json)
		self.create()

	# Getters / Setters
	def getStockName(self) :
		return self.stock_name

	def getAPI(self) :
		return self.api_key

	def getJson(self) :
		return self.json

	def getDate(self) :
		return self.date

	def getMoney(self) :
		return self.money

	def getAccount(self) :
		return self.account

	def getDayAgo(self) : # get the different values (day ago) of a stock  (open price, lowest price, highest price, volume, close price : dict)
			return self.day_ago

	def getWeekAgo(self) : # get the different values (day ago) of a stock  (open price, lowest price, highest price, volume, close price : dict)
			return self.week_ago

	def getMonthAgo(self) : # get the different values (day ago) of a stock  (open price, lowest price, highest price, volume, close price : dict)
			return self.day_ago

	def getDF(self) :
		return self.df

	def setDF(self, value) :
		self.df = value

	# Functions
	# Creation
	def createDates(self) :
		if(self.getDate() == None) :
			self.date = str(dt.datetime.strptime(list(getIntraday(self.getStockName(), self.getAPI()).keys())[0], "%Y-%m-%d %H:%M:%S").date())
		if(self.getDayAgo()==None or self.getMonthAgo()==None or self.getWeekAgo()==None) :
			DA, dDA, WA, dWA, MA, dMA = getStockOldValues(self.getStockName(), self.getAPI())
			if(DA!=None or WA!=None or MA!=None or dDA!=None or dWA!=None or dMA!=None) :
				self.day_ago = {'date' : dDA, 'open' : float(DA['1. open']), 'close' : float(DA['4. close']), 'high' : float(DA['2. high']), 'low' : float(DA['3. low'])}
				self.week_ago = {'date' : dWA, 'open' : float(WA['1. open']), 'close' : float(WA['4. close']), 'high' : float(WA['2. high']), 'low' : float(WA['3. low'])}
				self.month_ago = {'date' : dMA, 'open' : float(MA['1. open']), 'close' : float(MA['4. close']), 'high' : float(MA['2. high']), 'low' : float(MA['3. low'])}
			print("All dates were created.")
			return 0
		else :
			return -1

	def createJson(self) :
		data = {'stock' : self.getStockName(),'buy' : [], 'sell' : [], 'money' : self.getMoney(), 'amount' : 0 }
		if(not os.path.exists(self.getJson())) :
			f = open(self.getJson(),"w")
			json.dump(data, f)
			f.close()
			print('File {} created.'.format(self.getJson()))
		else :
			json_data = open(self.getJson())
			json_data = json.load(json_data)
			if(json_data.keys() == data.keys()) :
				print('File {} seems to be already exists.'.format(self.getJson()))
			else :
				return -1

	def create(self) :
		self.createDates()
		self.createJson()

	# Updating
	def update(self) :
		print("Updating the stock : {} -> {}.".format(self.getStockName(), self.getDate()))
		df = pd.DataFrame(data=getIntraday(self.getStockName(), self.getAPI())).transpose()
		df['day'] = pd.to_datetime(df.index).strftime('%Y-%m-%d')
		df = df[df['day'] == self.getDate()]
		df['time'] = pd.to_datetime(df.index).strftime('%H:%M:%S')
		df = df.astype({'1. open' : 'float64', '2. high' : 'float64', '3. low' : 'float64', '4. close' : 'float64', '5. volume' : 'int'})
		df = df.sort_values(by='time', ascending=True)
		df = df.reset_index(drop=True)
		df['id'] = df.index
		self.setDF(df)

	# Shwoing
	def show(self, buy_sell_pts=False, labels=True, line_labels=False, curve_fiting=False, interpolation=False, max_min=False, sockets=True, meanTrend=True, volumes=False) :
		print("Showing the stock price evolution : {}.".format(self.getStockName()))
		if(not self.getDF().empty) :
			df = self.getDF()
			ticks_range = np.arange(0, df.shape[0], 10)
			plt.figure(figsize=(19, 11))
			plt.title("Representation of {} stocks evolution the {}.".format(self.getStockName(),  self.getDate()))
			plt.xticks(ticks_range, df['time'][ticks_range], rotation='vertical')
			plt.ylim((df['3. low'].min()-1, df['2. high'].max()+1))
			sns.lineplot(x=df['id'], y=df['4. close'], markers=True, dashes=False)
			if(curve_fiting) :
				predict_x, predict_y, coefs = getCurvePolyFit(df['id'], df['4. close'], degree=15)
				sns.lineplot(x=predict_x, y=predict_y, markers=True, dashes=False) # Curve fitting
			if(interpolation) :
				interp_x, interp_y = getCurveInterp(df['id'], df['4. close'])
				sns.lineplot(x=interp_x, y=interp_y, markers=True, dashes=False) # Curve interpolation
			if(buy_sell_pts == True) :
				x_b, x_s = self.getAccount().getBuyingIds(), self.getAccount().getSellingIds()
				y_b = [float(df[df['id'] == item]['4. close'].values) for item in x_b]
				y_s = [float(df[df['id'] == item]['4. close'].values)  for item in x_s]
				plt.plot(x_b, y_b, 'rx') # plot buy points
				plt.plot(x_s, y_s, 'gx') # plot sell points
			if(labels) :
				plt.text(df['id'].iloc[-1] + 3, df['4. close'].iloc[-1], 'Last : '+str(df['4. close'].iloc[-1])) # Last
				plt.text(df['id'].iloc[-1] + 3, df['3. low'].min(), 'Lowest : '+str(df['3. low'].min())) # Lowest
				plt.text(df['id'].iloc[-1] + 3, df['2. high'].max(), 'Highest : '+str(df['2. high'].max())) #  Highest
				# plt.text(df['id'].iloc[-1] + 3, df['4. close'].iloc[-1]  - 0.2, 'Mean : '+str(df['4. close'].mean())) # Mean
			if(line_labels) :
				sns.lineplot(x=df['id'], y=df['3. low'].min()) # Lowest
				sns.lineplot(x=df['id'], y=df['2. high'].max()) # Highest
				sns.lineplot(x=df['id'], y=df['4. close'].mean()) # Mean
			if(max_min) :
				max_x, max_y = localMaximas(df['id'], df['4. close'], n_pts=1)
				min_x, min_y = localMinimas(df['id'], df['4. close'], n_pts=1)
				plt.plot(max_x, max_y, 'ro') # plot the local maximums
				plt.plot(min_x, min_y, 'bo') # plot the local minimums
			if(sockets) :
				mi, ma, _, _ = socketTrends(df)
				if mi.size!= 0:
					sns.lineplot(x=df['id'], y=mi.reshape(1, -1).tolist()[0], markers=True, dashes=False)
				if ma.size!= 0:
					sns.lineplot(x=df['id'], y=ma.reshape(1, -1).tolist()[0], markers=True, dashes=False)
			if(meanTrend) :
				mean_x, mean_y = meanTrend(df)
				sns.lineplot(x=mean_x, y=mean_y)
			if(volumes) :
				V_std = (df['5. volume'] - df['5. volume'].min()) / (df['5. volume'].max() - df['5. volume'].min()) * 2
				V_scaled = V_std * (df['4. close'].max() - df['4. close'].min()) + df['4. close'].min() - 1
				plt.text(df['id'].iloc[-1] + 3, V_scaled.max(), 'Max Volume : '+str(df['5. volume'].max())) # Last
				plt.text(df['id'].iloc[-1] + 3, V_scaled.min(), 'Min Volume : '+str(df['5. volume'].min())) # Lowest
				sns.lineplot(x=df['id'], y=V_scaled)
			plt.xlabel("Time")
			plt.ylabel("Stock price")
			plt.show()
		else :
			print("Can't draw the figure.")

	# Other function
	def buyStrategy(self, sell_threshold=0.002) :
		print("Trying to buy some "+self.getStockName()+" stocks.")
		df = self.getDF()
		if(self.getAccount().checkStock()) :
			buy_flag = False
		else :
			buy_flag = True
		previous_item = self.getDF()['1. open'].iloc[0]
		for index_item, item in df.iterrows() :
			if(index_item == 0) :
				pass
			else :
				r = np.arange(0, index_item+1, 1)
				max_x, max_y = localMaximas(df['id'][r].values, df['4. close'][r].values)
				min_x, min_y = localMinimas(df['id'][r].values, df['4. close'][r].values)
				if(buy_flag==True) : # If the buying_flag is set to True.
					if(previous_item<item['4. close']): # Ascending curve.
						if(min_y.size!=0 and (max_y.size==0 or max_x[-1]<min_x[-1])) : # The last local min/max is a minimum.
							if(buyLimit(time=item['time'])) : # If the last time to buy isn't overtaken.
								new_values_y_min, new_values_y_max, lr_min, lr_max = socketTrends(df.loc[df['id'].isin(r)])
								if(lr_min == 0 or new_values_y_min == np.array([])) :
									pass
								elif(item['4. close'] <= df.loc[df['id'].isin(r)]['4. close'].quantile(0.25) and item['4. close'] < lr_min.predict(np.array(item['id']).reshape(-1, 1))[0][0]) : # filter
									self.getAccount().buy(item)
									buy_flag = False
				elif(buy_flag==False) :
					if(previous_item>item['4. close']) : # Descending curve and self.getLastBuyingPrice()!=None
						if(max_y.size!=0 and (min_y.size==0 or max_x[-1]>min_x[-1])) : # The last local min/max is a maximum
							if(self.getAccount().checkStock()) : # We have something to sell and the buying_flag is set to False
								new_values_y_min, new_values_y_max, lr_min, lr_max = socketTrends(df.loc[df['id'].isin(r)])
								if(lr_max == 0 or new_values_y_max == np.array([])) :
									pass
								elif(item['4. close'] >= df.loc[df['id'].isin(r)]['4. close'].quantile(0.75) and item['4. close'] >= self.getAccount().getLastBuyingPrice() and item['4. close'] >= 0.85 * lr_max.predict(np.array(item['id']).reshape(-1, 1))[0][0]) : #filter
									self.getAccount().sell(item)
									buy_flag = True
				previous_item = item['4. close']

	# Test function
	def test(self) :
		print("Test on the stock : {}.".format(self.getStockName()))
		socketTrends(self.getDF())


# -*- coding: utf-8 -*-
# @Time    : 2022/12/7 10:49
# @Author  : jinjie
# @File    : test_app.py
import sqlite3
# 使用 unittest进行flask的单元测试

from flask import abort
import unittest
from myapp import app,db

# sqlite数据库配置
DATABASE = 'test.db'

# 使用测试类,工厂模式
class MyappTestCase(unittest.TestCase):
    def setUp(self) -> None:
        app.app_context()
        app.config.update(
        TESTING=True,
            WTF_CSRF_ENABLED=False,
            # 使用默认的sqlite数据库 # 单元测试使用内存级别的sqlite会更快
            SQLALCHEMY_DATABASE_URI=f'sqlite:///:{DATABASE}:'
        )
        db.creat_all()
        self.client = app.test_client()
        self.runner = app.test_cli_runner()

    def tearDown(self) -> None:
        db.session.remove()
        db.drop_all()

    #  测试用例
    def test_app_exist(self):
        self.assertFalse(app is None)

    def test_app_is_testing(self):
        self.assertTrue(app.config['TESTING'])

    def test_404_page(self):
        response = self.client.get('/nothing')
        data = response.get_data(as_text=True)
        self.assertIn('404 Error', data)
        self.assertIn('Go Back', data)
        self.assertEqual(response.status_code, 404)

    def test_500_page(self):
        # create route to abort the request with the 500 Error
        @app.route('/500')
        def internal_server_error_for_test():
            abort(500)

        response = self.client.get('/500')
        data = response.get_data(as_text=True)
        self.assertEqual(response.status_code, 500)
        self.assertIn('500 Error', data)
        self.assertIn('Go Back', data)

    def test_index_page(self):
        response = self.client.get('/')
        data = response.get_data(as_text=True)
        self.assertIn('flask is running', data)

if __name__ == '__main__':
    unittest.main()


from selenium import webdriver
from selenium.webdriver.common.keys import Keys

# add the chrome driver
chrome_driver ="C:\Development\chromedriver.exe"
driver = webdriver.Chrome(executable_path=chrome_driver)
# some web sites styles adjust the elements in the size screen and we maybe see less elements
driver.maximize_window()
driver.get("https://en.wikipedia.org/wiki/Main_Page")

# we need get the the article statistic, we can use xpath
statistics = driver.find_element_by_xpath('//*[@id="articlecount"]/a[1]')
print(statistics.text)
# or use any other methods
statistics = driver.find_element_by_css_selector("#articlecount a")
print(statistics.text)

# and now we wants to click in this element
#statistics.click()

# we can find any element by link text
wiki_portals = driver.find_element_by_link_text("All portals")
#wiki_portals.click()

# or search some in a search input
search = driver.find_element_by_name("search")
search.send_keys("Python", Keys.ENTER)


import cv2
import numpy as np
import matplotlib.pyplot as plt

img = cv2.imread("car.jpg", 0)

car = img[125:220, 200:300]  # roi of the image

indices = np.where(car != [0])
coordinates = zip(indices[0], indices[1])

# cv2.imshow('coordinates', coordinates)
plt.imshow(img)
plt.show()

cv2.waitKey(0)
cv2.destroyAllWindows()


import django
django.setup()
import datetime
import sys
import traceback
from apscheduler.schedulers.blocking import BlockingScheduler
from django.utils.timezone import now
from django.conf import settings
from logs.models import RedisSetting, RedisMetric
from utils.mail_helper import mail_admins
from utils.redis_scaling import redis_auto_scale
from utils.redis_scaling import check_frequency, collect_metrics, check_alarm

scheduler = BlockingScheduler()

@scheduler.scheduled_job('cron', second='0,30')
def check_scaling_metrics():
    print("Collecting redis metrics and checking for scaling...", now())
    cur_time = now()
    rss = RedisSetting.objects.all()
    for rs in rss:
        try:
            collect_valid = check_frequency(rs.metric_rate, cur_time)
            if collect_valid:
                collect_metrics(rs)
                check_alarm(rs)
        except Exception as e:
            print(e)
            mail_admins('Exception in collecting redis metrics',
                        'Could not collect metrics for redis ' + str(rs.redis_name) +
                        '\n' + str(traceback.format_exc()))

        try:
            scale_valid = check_frequency(rs.scaling_rate, cur_time, rs.enable_scaling)
            if scale_valid:
                redis_auto_scale(rs)
        except Exception as e:
            print(e)
            mail_admins('Exception in redis auto scaling',
                        'Could not perform scaling action for redis '
                        + str(rs.redis_name) + '\n' + str(traceback.format_exc()))

    return True

@scheduler.scheduled_job('cron', hour='*/2')
def clean_db_logs():
    print("Checking logs to clean..", file=sys.stderr)
    try:
        ft = now() - datetime.timedelta(days=settings.N_RETENTION_DAYS)
        rms = RedisMetric.objects.filter(sts__lte=ft)
        rms.delete()
    except Exception as e:
        print(e, file=sys.stderr)
        mail_admins("Exception in clean_db_logs cron", traceback.format_exc())

def main():
    try:
        print("Starting the blocking scheduler")
        scheduler.start()
    except KeyboardInterrupt:
        print('\nExiting by user request.\n', file=sys.stderr)
    except Exception:
        traceback.print_exc(file=sys.stdout)

    sys.exit(0)

if __name__ == '__main__':
    main()


from os import sep
import scrapy
import pandas as pd
import numpy as np
from scrapy.http import headers

class YapoSpider(scrapy.Spider):
    name = "yapo"
    df = pd.DataFrame(columns=['precio','Tipo de inmueble','Comuna','Dormitorios','Baños','Servicios'])

    def start_requests(self):
        urls = [
            #'https://www.yapo.cl/coquimbo/arriendo_temporada?ca=5_s&cg=1260&q=arriendo&o=1',
            'https://www.yapo.cl/coquimbo/arriendo_temporada?ca=5_s&q=arriendo&cg=1260&o=1',
            'https://www.yapo.cl/coquimbo/arriendo_temporada?ca=5_s&q=arriendo&cg=1260&o=2',
            'https://www.yapo.cl/coquimbo/arriendo_temporada?ca=5_s&q=arriendo&cg=1260&o=3',
            'https://www.yapo.cl/coquimbo/arriendo_temporada?ca=5_s&q=arriendo&cg=1260&o=4'
        ]
        for url in urls:
            yield scrapy.Request(url=url, callback=self.parse)

    def extract_data(self, res):
        for ad in res.css('tr.ad.listing_thumbs'):
            yield{
                'fecha': ad.css('span.date::text').get(),
                'hora' : ad.css('span.hour::text').get(),
                'link' : ad.css('a::attr(href)').get(),
                'anuncio': ad.css('a.title::text').get(),
                'precio': ad.css('span.price::text').get().strip(),
                'uf': ad.css('span.convertedPrice::text').get().strip(),
                'Region': ad.css('span.region::text').get(),
                'Comuna': ad.css('span.commune::text').get(),
                'tipo anuncio': ad.css('span.company_ad::text').get(),
            }

    def parse_ad(self, response):
        cell = {}
        precio=''
        current_ad = response.meta['current_ad']
        print(current_ad)
        try:
            precio = response.css('div.price.price-final strong::text').get().strip()
        except:
            precio = 'Sin informacion'

        for res in response.css('div.details table tbody'):
            cell = {
                'precio':precio,
                'anuncio':current_ad['anuncio'],
                'fecha':current_ad['fecha'],
                'hora':current_ad['hora'],
                'Region':current_ad['Region'],
                res.css('tr:nth-child(2) th::text').get():res.css('tr:nth-child(2) td::text').get(),
                res.css('tr:nth-child(3) th::text').get():res.css('tr:nth-child(3) td::text').get(),
                res.css('tr:nth-child(4) th::text').get():res.css('tr:nth-child(4) td::text').get(),
                res.css('tr:nth-child(5) th::text').get():res.css('tr:nth-child(5) td::text').get(),
                res.css('tr:nth-child(6) th::text').get():res.css('tr:nth-child(6) td::text').get(),
                res.css('tr:nth-child(7) th::text').get():res.css('tr:nth-child(7) td::text').get(),
                'Descripcion':response.css('div.description p::text').get().strip(),
                'link':response.request.url
            }

            try:
                self.df = pd.read_excel('yapo-testing.xlsx',engine='openpyxl',index_col=0)
            except:
                self.df = pd.DataFrame(columns=['precio','Tipo de inmueble','Comuna','Dormitorios','Baños','Servicios'])

            #sr = pd.Series(data=cell)
            #df_tmp = pd.DataFrame([list(data)], columns=columns)
            self.df = self.df[self.df.columns.drop(list(self.df.filter(regex='Unnamed:')))]

            self.df = self.df.append(cell,ignore_index=True)

            self.df.to_excel('yapo-testing.xlsx')

    def parse(self, response):
        for ad in response.css('tr.ad.listing_thumbs'):
            link = ad.css('a::attr(href)').get()
            current_ad = {
                'fecha': ad.css('span.date::text').get(),
                'hora' : ad.css('span.hour::text').get(),
                'Region': ad.css('span.region::text').get(),
                'anuncio': ad.css('a.title::text').get(),
            }
            yield scrapy.Request(url=link, callback=self.parse_ad, meta={'current_ad':current_ad})#yield

#scrapy crawl yapo --set DOWNLOAD_DELAY=3


import csv
import json

airspace_count = 0
fir_list = []
firpoint_list = None
firheader_list = None
# file_name = 'source/FIRBoundaries_test.dat'
# file_name = 'source/FIRBoundaries.dat'

def fir_header(header):
    global airspace_count, firheader_list
    airspace_count += 1
    firheader_list = header

def airspace_pnt(firpoint_list, point):
    coords = [float(point[0]), float(point[1])]
    coords.reverse()
    point = [coords]
    if firpoint_list == None:
        firpoint_list = point
    else:
        firpoint_list = firpoint_list + point
    return firpoint_list

def airspace_close():
    global firheader_list, firpoint_list
    header = firheader_list + [firpoint_list]
    keys = ['ICAO', 'IsOceanic', 'IsExtension', 'PointCount', 'MinLat', 'MinLon', 'MaxLat', 'MaxLon', 'CenterLat', 'CenterLon', 'Coordinates']
    fir_dict = dict(zip(keys, header))
    firpoint_list = None
    return fir_dict

def geojson_writer(features, output_dir):
    '''Create GeoJSON Header'''
    print(output_dir)
    basedict = {}
    crsdict = {}
    propertydict = {}
    feature_list = []
    basedict['type'] = "FeatureCollection"
    basedict['name'] = ''
    propertydict['name'] = "urn:ogc:def:crs:OGC:1.3:CRS84"
    crsdict['type'] = "name"
    crsdict['properties'] = propertydict
    basedict['crs'] = crsdict
    for section in features:
        feature_list = feature_list + section[1]

    basedict['features'] = feature_list

    '''write to text file'''
    print(output_dir)
    outputfile = open(output_dir, 'w')
    outputfile.write(json.dumps(basedict, indent=4))
    # print(json.dumps(basedict, indent=4))
    outputfile.close()
    print_string = 'GeoJSON saved to {}'.format(output_dir)
    print(print_string)

def dat_import(file_name, output_dir):
    global airspace_count, firpoint_list
    with open(file_name) as fir_file:
        csv_reader = csv.reader(fir_file, delimiter='|')
        for row in csv_reader:
            if len(row) > 2:
                if airspace_count > 0:
                    fir_list.append(airspace_close())
                fir_header(row)
            else:
                firpoint_list = airspace_pnt(firpoint_list, row)
        fir_list.append(airspace_close())

    features = []
    airspace_count = 0
    for item in fir_list:
        airspace_count += 1
        filename = 'firboundaries'
        section = [12582, [
            {'type': 'Feature', 'properties': {'id': airspace_count, 'ICAO': item['ICAO'], 'IsOceanic': item['IsOceanic'], 'IsExtension': item['IsExtension'], 'PointCount': item['PointCount'], 'MinLat': item['MinLat'], 'MinLon': item['MinLon'], 'MaxLat': item['MaxLat'], 'MaxLon': item['MaxLon'], 'CenterLat': item['CenterLat'], 'CenterLon': item['CenterLon']}, 'geometry': {'type': 'MultiPolygon', 'coordinates': [[item['Coordinates']]]}}], filename]
        features.append(section)

    geojson_writer(features, output_dir)


from PyPDF2 import PdfFileReader, PdfFileMerger
import eel
import time
from PIL import Image

# Initializing EEL into the web Directory
eel.init('F:\Sajed Project\Python\PDF Merger by using EEL\web')

# Function for Merging PDFs

@eel.expose
def pdfMaker(pdfList):
    if len(pdfList) > 1:
        try:
            merger = PdfFileMerger()
            print(pdfList)
            for pdf in pdfList:
                merger.append(rf'C:\Users\KN\Desktop\{pdf}')
            merger.write(rf'C:\Users\KN\Desktop\{pdf}(merged).pdf')
            eel.success()
        except Exception:

            print('choose pdf only')

            for pdf in pdfList:
                if not pdf.endswith('.pdf'):
                    pdfList.remove(pdf)
            print(pdfList)
            n = len(pdfList)
            eel.error(n)
            eel.otherFile(pdfList)

    else:
        if len(pdfList) == 0:
            eel.selectPDF()
        else:
            eel.selectTwoPDF()

# Function for converting PDFs

@eel.expose
def pdfConverter(imageList, value):
    if value == False:
        try:
            for image in imageList:
                image1 = Image.open(rf'C:\Users\KN\Desktop\{image}')
                im1 = image1.convert('RGB')
                im1.save(rf'C:\Users\KN\Desktop\{image}(conv).pdf')
                eel.successfull()
        except Exception:
            eel.imagesOnly()
    elif value == True:
        try:
            newList = []
            for image in imageList:
                image1 = Image.open(rf'C:\Users\KN\Desktop\{image}')
                im1 = image1.convert('RGB')
                newList.append(im1)
            firstImage = newList[0]
            del newList[0]
            firstImage.save(rf'C:\Users\KN\Desktop\conv.pdf',
                            save_all=True, append_images=newList)
            eel.successfull()
        except Exception:
            eel.imagesOnly()

@eel.expose
def onepdfConverter(imageList):
    try:
        myimage = imageList[0]
        image1 = Image.open(rf'C:\Users\KN\Desktop\{myimage}')
        im1 = image1.convert('RGB')
        im1.save(rf'C:\Users\KN\Desktop\{myimage}(conv).pdf')
        eel.successfull()
    except Exception:
        eel.imagesOnly()

@eel.expose
def chooseImages():
    eel.chooseImageFirst()

eel.start('F:\Sajed Project\Python\PDF Merger by using EEL\web\index.html')


import numpy as np
import matplotlib.pyplot as plt
import argparse

ap = argparse.ArgumentParser()
ap.add_argument("-f", "--motorfile", required=True, help="name of motor data file")
args = vars(ap.parse_args())

plt.rcParams['agg.path.chunksize'] = 20000

def ReadInsertionData():
    MotorFile = args["motorfile"]
    # PIDFile = args["pidfile"]
    prevRoll = 0
    prevRCM = 0
    prevtime = 0

    time = []
    vel = []
    RCM = []
    RCMVel = []
    EARoll = []
    EARollVel = []
    rcmVelandEARollVelTime = []

    with open(MotorFile) as f:
        lines = f.readlines()

        for i in range(len(lines)):
            line = lines[i]
            t = float(line.split()[0])
            velVal = float(line.split()[1])
            rcm = float(line.split()[2])

            time.append(t)
            vel.append(velVal)
            RCM.append(rcm)

    fig, axs = plt.subplots(2,1)
    axs[0].plot(time,vel,'k')
    axs[1].plot(time,RCM,'k')

    axs[0].set_ylabel(r'$V_{EA}$ (mm/s)', fontsize = 12)
    axs[1].set_ylabel(r'$\delta_{EA}$ (deg)', fontsize = 12)
    axs[1].set_xlabel("Time (s)", fontsize = 12)

    print("Max Speed: ", max(vel))
    print("Max RCM: ", max(RCM))

    plt.show()

if __name__ == '__main__':
	ReadInsertionData()


from datetime import datetime

from django.conf import settings
from django.db import models
from django.db.models import BooleanField
from django.db.models import CASCADE
from django.db.models import Case
from django.db.models import Count
from django.db.models import IntegerField
from django.db.models import Q
from django.db.models import SET_NULL
from django.db.models import When
from django.utils.translation import ugettext_lazy as _

from core.models import UserProfile

class TopicQuerySet(models.QuerySet):
    def listing_info(self, user):
        '''
        Adds information relevant to listing of topics
        '''

        topics = self
        now = datetime.now()

        if not user.is_anonymous:
            if not UserProfile.objects.get(user=user).topics_showall:
                topics = topics.filter(usertopic__user=user)

            # Annotate the user's favoriteness of topics. Note that even though
            # it's intended as a boolean, it is actually produced as an integer.
            # So it's 1/0, not True/False.
            if not user.is_anonymous:
                topics = topics.annotate(
                    favorited=Count(
                        Case(
                            When(usertopic__user=user, then=True),
                            output_field=BooleanField
                        ),
                        distinct=True
                    )
                )

        # Annotate issue count.
        topics = topics.annotate(issue_count=Count('issue', distinct=True))

        # Annotate usertopic count.
        topics = topics.annotate(usertopic_count=Count('usertopic', distinct=True))

        # Annotate counts of issues that are open and/or being voted on.
        topics = topics.annotate(
            issues_open=Count(
                Case(
                    When(issue__deadline_votes__gte=now, then=True),
                    output_field=IntegerField()
                ),
                distinct=True
            ),
            issues_voting=Count(
                Case(
                    When(Q(issue__deadline_votes__gte=now, issue__deadline_proposals__lt=now), then=True),
                    output_field=IntegerField()
                ),
                distinct=True
            )
        )

        return topics

class Topic(models.Model):
    """A collection of issues unified categorically."""
    objects = TopicQuerySet.as_manager()

    name = models.CharField(max_length=128, verbose_name=_('Name'))
    slug = models.SlugField(max_length=128, blank=True)

    description = models.TextField(verbose_name=_("Description"), null=True, blank=True)

    created_by = models.ForeignKey(
        settings.AUTH_USER_MODEL,
        editable=False,
        null=True,
        blank=True,
        related_name='topic_created_by',
        on_delete=SET_NULL
    )
    modified_by = models.ForeignKey(
        settings.AUTH_USER_MODEL,
        editable=False,
        null=True,
        blank=True,
        related_name='topic_modified_by',
        on_delete=SET_NULL
    )
    created = models.DateTimeField(auto_now_add=True)
    modified = models.DateTimeField(auto_now=True)

    polity = models.ForeignKey('polity.Polity', on_delete=CASCADE)

    class Meta:
        ordering = ["name"]

    def new_comments(self):
        return Comment.objects.filter(issue__topics=self).order_by("-created")[:10]

    def __str__(self):
        return u'%s' % (self.name)

class UserTopic(models.Model):
    """Whether a user likes a topic."""
    topic = models.ForeignKey('topic.Topic', on_delete=CASCADE)
    user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=CASCADE)

    class Meta:
        unique_together = (("topic", "user"),)


import pygame

pygame.init()

ourScreen=pygame.display.set_mode((400, 300))
pygame.display.set_caption("PYGAME")
finish = False;

color1 = (0,0,255)
color2 = (255,0,0)
color = color1

# coordinates
x=30
y=30

# frame per sec
#clock = pygame.time.Clock()

while not finish:
    for e in pygame.event.get():
        if e.type == pygame.QUIT:
            finish = True

        #pygame.draw.rect(ourScreen, (0, 0, 255), pygame.Rect(100,100,60,60))

        #if e.type == pygame.KEYDOWN and e.key == pygame.K_SPACE:
        #    if color==color1:
        #       color=color2
        #    else: color=color1
        if e.type == pygame.KEYDOWN:
            if e.key == pygame.K_UP :     y-=3
            elif e.key == pygame.K_DOWN : y+=3
            elif e.key == pygame.K_LEFT : x-=3
            elif e.key == pygame.K_RIGHT: x+=3

        if color==color1:color=color2
        else: color=color1

        #ourScreen.fill(color3)

        pygame.draw.rect(ourScreen, color, pygame.Rect(x,y,60,60))

        pygame.display.update()


from torch import nn
from sklearn.metrics import mean_squared_error
import math
import pandas as pd
import numpy as np
import torch
from torch.utils.data import DataLoader, TensorDataset
import os

def load_prediction(folder, model_name):
    '''
    Load one prediction
    '''
    return np.loadtxt(os.path.join(folder, model_name + '.txt'))

def load_all_predictions(**args):
    '''
    Load prediction for all of the folds of all models to be blended
    '''
    cv_folder = args.get('cv_folder')
    all_predictions = {}
    for model_type, model_name in zip(args.get('model_types_blending'), args.get('model_names_blending')):
        print('Loading predictions for model: {}'.format(model_type))
        all_predictions[model_type] = []
        for i in range(args.get('k_fold')):
            all_predictions[model_type].append(load_prediction(cv_folder, model_name + '_fold_' + str(i+1)))
    return all_predictions

def init_weights(layers, init_type):
    '''
    Initialize weights for a NN layer. if init_type is not None, it should be either
    "xavier" (https://proceedings.mlr.press/v9/glorot10a/glorot10a.pdf) or
    "kaiming" (https://pytorch.org/docs/stable/_modules/torch/nn/init.html#kaiming_uniform_)
    '''
    for layer in layers:
        if init_type == 'xavier':
            size = layer.weight.size()
            fan_out = size[0]
            fan_in = size[1]
            std = np.sqrt(2.0/(fan_in + fan_out))
            layer.weight.data.normal_(0.0, std)
        elif init_type == 'kaiming':
            nn.init.kaiming_normal_(layer.weight)

        # Normal Initialization for Biases
        layer.bias.data.normal_(0.0, 0.001)

def get_activation(activation_type):
    '''
    Activation function wrapper. One of ('relu', 'leakyrelu', 'sigmoid', 'tanh')
    '''
    if activation_type == 'relu':
        return nn.ReLU()
    elif activation_type == 'leakyrelu':
        return nn.LeakyReLU()
    elif activation_type == 'sigmoid':
        return nn.Sigmoid()
    elif activation_type == 'tanh':
        return nn.Tanh()

def get_list(n):
    '''
    Convert an object to list, used for NCF layers
    '''
    if isinstance(n, (int, float)):
        return [n]
    elif hasattr(n, '__iter__'):
        return list(n)
    raise TypeError('layers configuraiton should be a single number or a list of numbers')

def get_all_user_movie_indices(num_users, num_movies):
    '''
    Get all index combinations of users in [0, num_users) and movies in [0, num_movies)
    '''
    full_matrix_users = []
    for i in range(num_users):
        for j in range(num_movies):
            full_matrix_users.append(i)
    full_matrix_movies = []
    for i in range(num_users):
        for j in range(num_movies):
            full_matrix_movies.append(j)
    return full_matrix_users, full_matrix_movies

def get_dataloader(users_train, movies_train, ratings_train, impute=0, **args):
    '''
    Convert available rating records to a num_users*num_movies matrix, return its pytorch dataloder,
    data torch tensor, mask tensor, and user id tensor
    '''
    num_users = args.get('num_users')
    num_movies = args.get('num_movies')
    device = args.get('device')
    data_zeros = np.full((num_users, num_movies), impute)
    data_mask = np.full((num_users, num_movies), impute)

    for i, (user, movie) in enumerate(zip(users_train, movies_train)):
        data_zeros[user][movie] = ratings_train[i]
        data_mask[user][movie] = 1

    data_torch = torch.tensor(data_zeros, device=device).float()
    mask_torch = torch.tensor(data_mask, device=device)
    user_id_torch = torch.tensor(list(range(0, num_users)), device=device)
    dataloader = DataLoader(TensorDataset(data_torch, mask_torch, user_id_torch), batch_size=args.get('batch_size'), shuffle=True)
    return dataloader, data_torch, mask_torch, user_id_torch

rmse = lambda x, y: math.sqrt(mean_squared_error(x, y))
def get_score(predictions, target_values):
    '''
    Get RMSE score between predictions and ground truth
    '''
    return rmse(predictions, target_values)

def extract_prediction_from_full_matrix(reconstructed_matrix, users, movies):
    '''
    returns predictions for the users-movies combinations specified based on a full m*n matrix
    '''
    assert(len(users) == len(movies)), "users-movies combinations specified should have equal length"
    predictions = np.zeros(len(users))

    for i, (user, movie) in enumerate(zip(users, movies)):
        predictions[i] = reconstructed_matrix[user][movie]

    return predictions

def extract_user_movie_rating_arrays(df):
    '''
    Convert dataframe in the format of 'rx_cy' rows into lists of users, movies, ratings
    '''
    users = list()
    movies = list()
    for user, movie in df.Id.str.extract('r(\d+)_c(\d+)').values:
        users.append(int(user) - 1)
        movies.append(int(movie) - 1)
    users = np.array(users)
    movies = np.array(movies)
    ratings = df['Prediction'].values
    return users, movies, ratings

def stand_norm(A):
    """
    Normalize a matrix such that it has mean 0 and variance 1, columnwise
    """
    col_mean = np.nanmean(A, axis=0)
    col_std = np.nanstd(A, axis=0)
    A_n = (A-col_mean)/col_std
    return A_n, col_mean, col_std

def de_norm(A, col_mean, col_std):
    """
    Denormalize a matrix
    """
    return (A*col_std)+col_mean

def impute_matrix(A, impute_method="zero"):
    """
    Impute missing values of a matrix.
    impute_method is either "zero" for zero imputation or "col_mean" for columnwise mean imputation
    """
    if impute_method == "zero":
        A_i = np.nan_to_num(A, nan=0.0)
        return A_i
    else:
        A_i = np.copy(A)
        item_mean = np.nanmean(A_i, axis=0)
        na_idx = np.where(np.isnan(A_i))
        A_i[na_idx] = np.take(item_mean, na_idx[1])
        return A_i


import itertools
def solution(numbers):

    answer = set()

    arr = list(itertools.combinations(numbers,2))
    print(arr)

    for a in arr:
        answer.add(sum(a))

    answer = list(answer)
    answer.sort()

    return answer

프로그래머스/lv1/68644. 두 개 뽑아서 더하기/두 개 뽑아서 더하기.py

두 개 뽑아서 더하기.py


import xml.etree.ElementTree as ET

class SubjectClass():
    def __init__(self, name, Tasks, TD):
        self.Name = name
        self.Task = int(Tasks)
        self.TaskDone = int(TD)

class student():
    Surname = str()
    Midame = str()
    ID = str()
    SubjectList = []
    def __init__(self, name, surname, midname, ID):
        self.Name = name
        self.Surname = surname
        self.Midame = midname
        self.ID = ID
        self.SubjectList = []

class group():
    def __init__(self, Name):
        self.Name = Name
        self.GroupList = []

tree = ET.parse('C:/PyProgs/Lab3/tsst2.xml')
root = tree.getroot()
SuperList = [group(Groups.get("GNum")) for Groups in root.findall("Groups/Group")]
for GroupObj in SuperList:
    Formattext = "Groups/Group[@GNum='{0}']/Student".format(GroupObj.Name)
    GroupObj.GroupList = [student(Students.get("name"), Students.get("surname"), Students.get("midname"), Students.get("ID")) for Students in root.findall(Formattext)]
    for StudObj in GroupObj.GroupList:
        Formattext2 = "Groups/Group[@GNum='{0}']/Student[@ID='{1}']/Subject".format(GroupObj.Name, StudObj.ID)
        StudObj.SubjectList = [SubjectClass(Subjects.get('SName'), Subjects.find("Tasks").text, Subjects.find("Tasks_Done").text) for Subjects in root.findall(Formattext2)]
print('2.1')
for GroupObj in SuperList:
    print(GroupObj.Name,': ')
    for Studobj in GroupObj.GroupList:
        print(Studobj.Surname, Studobj.Name, Studobj.Midame)

print('2.2, 2.3, 2.4')
print(SuperList[0].GroupList[2].Surname)
TaskAll = 0
TaskDoneAll = 0
for SubjObj in SuperList[0].GroupList[2].SubjectList:
   PercentSub = SubjObj.TaskDone/SubjObj.Task*100
   print(SubjObj.Name,': ', SubjObj.Task,'работ сделано', PercentSub,'Процент готовности по предмету')
   TaskAll = TaskAll + SubjObj.Task
   TaskDoneAll = TaskDoneAll + SubjObj.TaskDone
print(TaskDoneAll/TaskAll*100,' общий процент готовности')

print(3)
print(SuperList[0].GroupList[2].Surname)
for SubjObj in SuperList[0].GroupList[2].SubjectList:
   if SubjObj.TaskDone/SubjObj.Task*100 > 50 :
    print(SubjObj.Name,': ', SubjObj.Task,'работ сделано', PercentSub,'Процент готовности по предмету')


import numpy as np
import torch
import pytest
from torch import tensor, cuda

from mrphy import γH, dt0, π
from mrphy import beffective, slowsims

class Test_slowsims:

    device = torch.device('cuda' if cuda.is_available() else 'cpu')
    # device = torch.device('cpu')
    # dtype, atol = torch.float32, 1e-4
    dtype, atol = torch.float64, 1e-9
    print(device)

    dkw = {'dtype': dtype, 'device': device}

    γ = γH.to(**dkw)  # Hz/Gauss
    dt = dt0.to(**dkw)  # Sec

    @staticmethod
    def np(x):
        return x.detach().cpu().numpy()

    def test_blochsims(self):

        dkw, atol = self.dkw, self.atol
        γ, dt = self.γ, self.dt

        # spins  # (1,nM,xyz)
        M0 = tensor([[[1., 0., 0.], [0., 1., 0.], [0., 0., 1.]]], **dkw)
        N, nM, nT = M0.shape[0], M0.shape[1], 512
        Nd = (nM,)

        # parameters: Sec; cm.
        T1, T2 = tensor([[1.]], **dkw), tensor([[4e-2]], **dkw)

        E1, E2, γ2πdt = torch.exp(-dt/T1), torch.exp(-dt/T2), 2*π*γ*dt
        E1_1 = E1 - 1

        loc_x = torch.linspace(-1., 1., steps=nM, **dkw).reshape((N,)+Nd)
        loc_y = torch.linspace(-1., 1., steps=nM, **dkw).reshape((N,)+Nd)
        loc_z = torch.ones((N,)+Nd, **dkw)
        loc = torch.stack([loc_x, loc_y, loc_z], 2)  # (1,nM,xyz)

        Δf = -loc_x * γ  # gr_x==1 Gauss/cm cancels Δf
        b1Map = tensor([1., 0.], **dkw).reshape((N, 1, 2, 1))

        # pulse: Sec; Gauss; Gauss/cm.
        pulse_size = (N, 1, nT)
        t = torch.arange(0, nT, **dkw).reshape(pulse_size)
        rf = 10*torch.cat([torch.cos(t/nT*2*π),              # (1,xy,nT,nCoils)
                           torch.sin(t/nT*2*π)], 1)[..., None]
        gr = torch.cat([torch.ones(pulse_size, **dkw),
                        torch.zeros(pulse_size, **dkw),
                        10*torch.atan(t - round(nT/2))/π], 1)  # (1,xyz,nT)

        rf.requires_grad, gr.requires_grad = True, True

        beff = beffective.rfgr2beff(rf, gr, loc, Δf=Δf, b1Map=b1Map, γ=γ)

        A, B = beffective.beff2ab(beff, E1=E1, E2=E2, γ=γ, dt=dt)

        # sim
        Mo1 = slowsims.blochsim(M0, beff, T1=T1, T2=T2, γ=γ, dt=dt)

        Mo2, Mo_tmp = M0.clone(), M0.clone()
        for t in range(nT):
            Mo2, _ = slowsims.blochsim_1step(Mo2, Mo_tmp, beff[..., t, :],
                                             E1, E1_1, E2, γ2πdt)

        Mo3 = slowsims.blochsim_ab(M0, A, B)

        # assertion
        Mo0 = np.array(
            [[[0.559535641648385,  0.663342640621335, 0.416341441715101],
              [0.391994737048090,  0.210182892388552, -0.860954821972489],
              [-0.677062008711222, 0.673391604920576, -0.143262993311057]]])
        ref = pytest.approx(Mo0, abs=atol)

        f1, f2, f3 = (self.np(x) == ref for x in (Mo1, Mo2, Mo3))
        assert(f1 and f2 and f3)

        # Verify gradients can chain rule back to `rf` and `gr`
        foo = torch.sum(Mo1)
        foo.backward(retain_graph=True)  # keep graph to check `bar.backward()`
        rf_grad1, gr_grad1 = self.np(rf.grad), self.np(gr.grad)
        rf.grad = gr.grad = None  # clear grads from `foo`

        bar = torch.sum(Mo3)
        bar.backward()
        rf_grad2, gr_grad2 = self.np(rf.grad), self.np(gr.grad)
        assert(rf_grad1 == pytest.approx(rf_grad2, abs=atol))
        assert(gr_grad1 == pytest.approx(gr_grad2, abs=atol))

        return

    def test_freeprec(self):

        dkw, atol = self.dkw, self.atol

        # spins  # (1,nM,xyz)
        Mi = tensor([[[1., 0., 0.], [0., 1., 0.], [0., 0., 1.]]], **dkw)

        E1, E2 = tensor([[0.5]], **dkw), tensor([[0.5]], **dkw)  # (1,1)

        # scalar dur, T1, T2
        dur = torch.tensor(0.5, **dkw)
        T1, T2 = -dur/torch.log(E1), -dur/torch.log(E2)  # ()

        Δf = tensor([[1/4/dur, -1/4/dur, 1]], **dkw)  # (1, nM) quater-circle

        Mo = slowsims.freeprec(Mi, dur, T1=T1, T2=T2, Δf=Δf)

        Mo0 = np.array([[[0., -0.5, 0.5], [-0.5, 0, 0.5], [0., 0., 1.]]])
        Mref = pytest.approx(Mo0, abs=atol)

        assert(self.np(Mo) == Mref)
        return

if __name__ == '__main__':
    tmp = Test_slowsims()
    tmp.test_blochsims()
    tmp.test_freeprec()


import re
import time

from bs4 import BeautifulSoup
from scrapy.http import Request
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from scrapy_statis.items import Item

from scrapy_statis.settings import year_settings

if year_settings == "all":
    year_pat = r'[0-9]{1,4}'
else:
    year_pat = '%s' % year_settings

class StatisGovSpider(CrawlSpider):
    name = 'stats_spider'
    allowed_domains = ['stats.gov.cn']
    start_urls = ['http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/']
    rules = (
        # 获取各年份统计数据链接
        Rule(LinkExtractor(
            allow=(r'http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/%s/index\.html' % year_pat)),
            callback='parse_province'),)

    def parse_province(self, response):
        """
        :param response: 各年份各省的链接
        :return: 【年份、省,省代码、省链接】四个维度
        """
        print("启动采集程序成功！")
        print("开始采集数据了！")

        # 定义一个空列表，保存本级解析页面获取到的【年份、省,省代码、省链接】，items_1元素是一个各省数据的列表
        items_1 = []
        while response.status != 200:
            time.sleep(10)
            self.parse_main(response)
        for prov in LinkExtractor(allow=(r'tjyqhdmhcxhfdm/%s/\d\d\.html' % year_pat)).extract_links(response):
            """
            实例化items.py中定义的item！
            注意：必须在循环中实例化！
            """
            item = Item()
            url = prov.url
            text = prov.text
            item['year'] = url[-12:-8]
            item['prov'] = text
            item['prov_code'] = url[-7:-5]
            item['prov_url'] = url

            items_1.append(item)

        for item_tmp in items_1:

            yield Request(item_tmp['prov_url'], meta={'item_1': item_tmp}, callback=self.parse_city,
                          dont_filter=True)

    def parse_city(self, response):
        """
        :param response: 解析各省城市列表
        :return 【城市名, 城市代码, 城市区划代码, 城市链接】四个维度
        """
        # 接收上级页面解析数据
        item_1 = response.meta['item_1']
        # 定义一个空列表，接收parse_prov中解析的数据，并保存本级页面解析的数据,items_2元素是一个各城市数据的列表
        items_2 = []
        # 获取当前省城市列表text
        text = response.xpath('/html/body/table[2]/tbody/tr[1]/td/table/tbody/tr[2]/td/table/tbody/tr/td/table') \
            [0].extract()
        # 使用正则表达式解析各城市数据，保存在items_2中
        pat = r'(\d{12})	([\u4e00-\u9fa5]+)'
        for prov_code, city_code, city_stats_code, city in re.findall(pat, text):
            # 实例化本层的item, 注意：必须在循环中实例化！！！
            item = Item()
            # 将上级页面的item_1中的【年份、省,省代码、省链接】传递给当前item：
            for key in item_1:
                item[key] = item_1[key]
            # 将本级页面解析出的【城市名, 城市代码, 城市区划代码, 城市链接】放入item字典
            item['city'] = city
            item['city_code'] = city_code
            item['city_stats_code'] = city_stats_code
            item['city_url'] = 'http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/%s/%s/%s.html' \
                               % (item['year'], item['prov_code'], item['city_code'])
            items_2.append(item)

        for item in items_2:
            yield Request(item['city_url'], meta={'item_2': item}, callback=self.parse_district, dont_filter=True)

    def parse_district(self, response):

        """
        :param response: 解析各城市区县列表
        :return: 【区县名，区县行政代码，区县区划代码、区县临时编号，区县链接】
        """
        # 接收上级页面解析数据
        item_2 = response.meta['item_2']
        # 定义一个空列表，接收parse_city中解析的数据，并保存本级页面解析的数据,items_3元素是一个各城市数据的列表
        items_3 = []
        # 获取区县列表text
        soup = BeautifulSoup(response.text, 'lxml')

        district_dict = {}
        for link in soup.find_all(href=re.compile("\d\d/\d{1,10}\.html")):
            href = link['href']
            district_dict[href] = {'code': '', 'name': ''}

        for link in soup.find_all(href=re.compile("\d\d/\d{1,10}\.html")):
            href = link['href']
            district = link.string
            m = re.search('[\u4e00-\u9fa5]', district)
            if m is not None:
                district_dict[href]['name'] = district
            else:
                district_dict[href]['code'] = district

        for href, district_info in district_dict.items():
            tmpcode, district_code = href.strip('.html').split(r'/')
            district, stats_code = district_info['name'], district_info['code']
            item = Item()
            """
            将上级页面中的item传递给本层item，包括：
            【年份、省,省代码、省链接】+ 【城市名, 城市代码, 城市区划代码, 城市链接】
            """
            for key in item_2:
                item[key] = item_2[key]
            # 解析本级页面【区县名，区县行政代码，区县区划代码、区县临时编号，区县链接】并传递给item
            item['district'] = district
            item['district_code'] = district_code
            item['district_stats_code'] = stats_code
            item['district_tmp_code'] = tmpcode
            item['district_url'] = 'http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/%s/%s/%s/%s.html' \
                                   % (item['year'], item['prov_code'], tmpcode, item['district_code'])

            items_3.append(item)

        for item in items_3:
            if len(item['district_code']) == 6:
                yield Request(item['district_url'], meta={'item_3': item}, callback=self.parse_town, dont_filter=True)
            else:
                # 对于类似东莞等无区县城市的处理
                item['town_code'] = item['district_code']
                item['town'] = item['district']
                item['town_stats_code'] = item['district_stats_code']
                item['town_code'] = item['district_code']
                item['district_code'] = "%s00" % item['city_code']
                item['town_url'] = item['district_url']
                yield Request(item['district_url'], meta={'item_4': item}, callback=self.parse_village,
                              dont_filter=True)

    def parse_town(self, response):
        """
        :param response: 解析各区县下各乡镇或者街道办事处列表
        :return:【乡镇名或办事处名，town行政代码、town统计区划代码、town临时代码】
        """
        # 定义一个字典对象item_3接收上级页面解析数据
        item_3 = response.meta['item_3']

        text = response.xpath('/html/body/table[2]/tbody/tr[1]/td/table/tbody/tr[2]/td/table/tbody/tr/td/table') \
            [0].extract()
        pat = r'(\d{12})	([\u4e00-\u9fa5]+)'
        for town_tmp_code, town_code, town_stats_code, town in re.findall(pat, text):

            # 实例化item
            item = Item()
            # 将上级页面解析结果传递给本层item
            for key in item_3:
                item[key] = item_3[key]
            # 解析本级页面数据
            item['town'] = town
            item['town_code'] = town_code
            item['town_stats_code'] = town_stats_code
            # 生成下级页面解析链接
            item['town_url'] = 'http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/%s/%s/%s/%s/%s.html' \
                               % (item['year'], item['prov_code'], item['district_tmp_code'],
                                  town_tmp_code, item['town_code'])

            yield Request(item['town_url'], meta={'item_4': item}, callback=self.parse_village, dont_filter=True)

    def parse_village(self, response):
        """
        :param response: 解析最后一级社区列表
        :return: 【社区名，社区城乡划分代码、社区行政区划代码】
        """
        item_4 = response.meta['item_4']

        text = response.xpath('/html/body/table[2]/tbody/tr[1]/td/table/tbody/tr[2]/td/table/tbody/tr/td/table')[
            0].extract()

        item = Item()
        for key in item_4:
            item[key] = item_4[key]

        item['village_info'] = text

        yield item


import start_main
import sys
import time

from tkinter import *

from PySide6.QtWidgets import QApplication, QVBoxLayout, QMainWindow, QWidget, QPushButton, QLabel
from PySide6.QtGui import QIcon
from PySide6.QtCore import Qt, QTimer, Slot

class Start(QMainWindow):
    '''
    This is the main window that appears when we start the application. This window
    serves no utility purpose and is only for the display of the program's logo.
    '''
    def __init__(self):
        super().__init__()

        self.w = None # No external window yet.

        QTimer.singleShot(2000, self.show_next_instance)

        self.setWindow()
        self.setWindowTitle("oof")

    def show_next_instance(self):
        if self.w is None:
            self.w = start_main.ZenDocs()
            self.close()
        self.w.showMaximized()

    '''Sets the layout of the window'''
    def setWindow(self):
        title = "Zendocs"
        icon = "img/icon2.png"

        stylesheet = "background-image: url(./img/intrologo.jpg);" \
                        "background-repeat: no-repeat;" \
                        "background-position: center"

        root = Tk()
        height = root.winfo_screenheight()
        width = root.winfo_screenwidth()

        startTop, startLeft = height//4, width//4

        self.setWindowTitle(title)
        self.setWindowIcon(QIcon(icon))
        self.setStyleSheet(stylesheet)

        self.setGeometry(startLeft, startTop, width//2, height//2)

        self.setWindowFlag(Qt.FramelessWindowHint)

def main():
    # Initialize application
    app = QApplication([])

    # Initialize widget
    mainWindow = Start()

    # Show the window
    mainWindow.show()

    # Exit
    sys.exit(app.exec())

if __name__ == "__main__":
	main()


from typing import List, Any
from random import random
import math
import numpy as np
from sklearn.decomposition import PCA
from math import sqrt

d = 50

def CalcDistant(vec1, vec2):
    vec = np.multiply(np.array(vec1 - vec2), np.array(vec1 - vec2))
    return vec.sum()

#text is a tokenized list of words
def vectorization(text, embed):
    vec = np.zeros((1, d))
    num = 0;
    unknown = 0
    for i in text:
        if i in embed:
            vec += embed[i]
    return vec/np.linalg.norm(vec)

# preprocessing: lowering and splitting
def preprocessing(text):
    text = text.lower()
    stripped_text = ''

    for c in text:
        if ((not c.isalnum()) and (c != ' ') and (len(c.encode(encoding='utf_8')) == 1)):
            stripped_text = stripped_text + ' ' + c + ' '
        else:
            stripped_text += c
    return stripped_text

#tokenization: getting stuff together
def tokenization(text):
    q = text.split(' ')
    while ('' in q):
        q.remove('')
    while ('>' in q):
        q.remove('>')
    while ('<' in q):
        q.remove('<')
    return q

def init_params(layer_sizes,activation):
    d = {}
    for i in range(len(layer_sizes) - 1):
        if ((activation == "sigmoid")|(activation == "tanh")):
            #Xavier
            w = np.random.randn((layer_sizes[i]+1)*layer_sizes[i+1]) * sqrt(1.0/(layer_sizes[i]+1)*layer_sizes[i+1])
            w = np.reshape(w, (layer_sizes[i]+1, layer_sizes[i+1]))
            d[i] = w
        else:
            #He
            w = np.random.randn((layer_sizes[i]+1)*layer_sizes[i+1]) * sqrt(2.0/(layer_sizes[i]+1)*layer_sizes[i+1])
            w = np.reshape(w, (layer_sizes[i]+1, layer_sizes[i+1]))
            d[i] = w

    #last layer
    if ((activation == "sigmoid")|(activation == "tanh")):
            #Xavier
        w = np.random.randn((layer_sizes[-1]+1)*2) * sqrt(1.0/(2*(layer_sizes[-1]+1)))
        w = np.reshape(w, (layer_sizes[-1]+1, 2))
        d[len(layer_sizes) - 1] = w
    else:
            #He
        w = np.random.randn((layer_sizes[-1]+1)*2) * sqrt(2.0/(2*(layer_sizes[-1]+1)))
        w = np.reshape(w, (layer_sizes[-1]+1, 2))
        d[len(layer_sizes) - 1] = w
    return d

def tanh(weight, x): #X is a matrix, weight is a matrix - return matrix
    b = np.dot(x, weight)
    b = 1 + np.exp(-2*b.T)
    return np.array(2/b) - 1

def relu(weight, x): #X is a matrix, weight is a matrix  - return matrix
    b = np.dot(x, weight)
    return np.maximum(b.T, 0)

def sigmoid(weight, x): #X is a matrix, weight is a matrix - return matrix
    b = np.dot(x, weight)
    b = 1 + np.exp(-b.T)
    return np.array(1/b).T

def fully_connected(a_prev, W, activation):
    a_prev = np.insert(a_prev, 0, np.ones(a_prev.shape[0]), 1)
    if (activation == "sigmoid"):
        layer_out = sigmoid (W, a_prev)
    elif (activation == "tanh"):
        layer_out = tanh (W, a_prev)
    elif (activation == "relu"):
        layer_out = relu (W, a_prev)
    else:
        #linear
        layer_out = np.dot(a_prev, W)
        layer_out = layer_out

    return layer_out.T, a_prev, W

def ffnn(X, params, activation):
    cash = []
    for i in params:
        X, tmpcash1, tmpcash2 = fully_connected(X, params[i], activation)
        cash.append (tmpcash1)
        cash.append (tmpcash2)
    X = np.exp(X - np.amax(X, axis = 0))
    X = (X.T/X.sum(axis = 1)).T #softmax
    cash.append (X)
    return X, cash

def softmax_crossentropy(ZL, Y):
    Y = np.array(Y)
    log_likelihood = -np.log(np.dot(ZL.T, Y))
    loss = np.sum(log_likelihood) / len(Y)
    math_expect = np.sum(np.dot(ZL.T, Y))
    return loss

def fully_connected_backward(dA, cache, activation):
    if (activation == "sigmoid"):
        derivative = cache[2]*(1-cache[2])
    elif (activation == "tanh"):
        derivative = 1 - cache[2]*cache[2]
    elif (activation == "relu"):
        derivative = np.maximum(cache[2], 0)
    else:
        #linear
        derivative = cache[2]
    if (dA.shape[1] == derivative.shape[0]):
        tmp = np.dot(dA,derivative)
    else:
        tmp = np.dot(dA.T,derivative)
    check_gradients(activation, cache[2], derivative)
    DZ = np.dot (cache[3], dA.T)*derivative.T
    DW = np.dot(cache[0].T, DZ.T)
    return np.delete(DZ.T, (0), axis = 1), DW

def ffnn_backward(dZL, caches, activation):
    d_param = {}
    dA = dZL
    d_param[-int(len(caches)/2)] = dA
    d_param[int(len(caches)/2)] = np.dot(caches[len(caches) - 4].T,dA)
    for i in range (int(len(caches)) - 4, 0, -2):
        d_param[- int(i/2) - 1], d_param[int(i/2) + 1] = fully_connected_backward(dA, caches[i-2:i+3], activation)
        dA = d_param[- int(i/2) - 1]
    #positive/2 - DZ
    #negative/2 - DW
    return d_param

def softmax_crossentropy_backward(cache):
    dZL = cache[len(cache) - 2] - cache[len(cache) - 1] # ^Y - Y
    return np.array(dZL)

def check_gradients (activation, x, grad):
    h = 0.00001
    if (activation == "sigmoid"):
        correct = (1/(1 + np.exp(-(x + h).T)) - 1/(1 + np.exp(-(x - h).T)))/(2*h)
    elif (activation == "tanh"):
        correct = (2/(1 + np.exp(-2*(x + h).T)) - 2/(1 + np.exp(-2*(x - h).T)))/(2*h)
    elif (activation == "relu"):
        correct = (np.maximum(x + h, 0) - np.maximum(x - h, 0))/(2*h)
    else:
        #linear
        correct = 1
    print ("error = ", np.sum(abs(correct - grad.T).sum(axis = 0)/correct.shape[1]))
    return

def gd_step(params,grads,learning_rate, alpha):
    alpha = 0.0001
    for i in range (len(params)):
        if (params[i].shape[1] != grads[i+2].shape[1]):
            params[i] = params[i] - np.delete(learning_rate*grads[i+2], (0), axis=1) + alpha*params[i]
        else:
            params[i] = params[i] - learning_rate*grads[i+2] + alpha*params[i]
    return params

def train_ffnn(Xtrain, Ytrain, Xdev, Ydev, layer_sizes, learning_rate, num_epochs, batch_size):
    Xtrain -= Xtrain.mean(axis = 0)
    Xtrain /= Xtrain.std(axis = 0)
    X = np.zeros((Ytrain.shape[0], Ytrain.shape[1]))
    print(Ytrain)
    dict_weights = init_params(layer_sizes,"tanh")
    cur_batch = 0
    alpha = 0.001
    for i in range (num_epochs):
        if ((i == 500)|(i == 700)|i == 900):
            learning_rate /= 2
            alpha /= 2
        if (cur_batch + batch_size > Xtrain.shape[0]):
            cur_batch = 0

        #X[cur_batch:cur_batch + batch_size], cache = ffnn(np.insert(Xtrain[cur_batch:cur_batch + batch_size], 0, np.ones(batch_size), 1), dict_weights,"tanh")
        X[cur_batch:cur_batch + batch_size], cache = ffnn(Xtrain[cur_batch:cur_batch + batch_size], dict_weights,"tanh")
        cache.append(np.array(Ytrain[cur_batch:cur_batch + batch_size]))

        print (i, "] precision = ", np.sum(abs((X[cur_batch:cur_batch + batch_size]-Ytrain[cur_batch:cur_batch + batch_size]))/(2*batch_size)))

        dict_grads = ffnn_backward(softmax_crossentropy_backward(cache),cache,"tanh")
        #for i in (dict_grads.keys()):

        dict_weights = gd_step(dict_weights, dict_grads, learning_rate, alpha)
        cur_batch += batch_size

    return dict_weights

def train(
        train_texts: List[str],
        train_labels: List[str],
        pretrain_params: Any = None) -> Any:

    embeddings = {}
    f = open('glove.6B.50d.txt','r')
    for line in f:
        k = line.index(' ')
        embeddings[line[:k]] = np.fromstring(line[k:], sep = ' ')
    matr = np.ones((1, d))

    doc_amount = len(train_texts) # amount of documents in train sample
    y_labels = []
    for i in range(doc_amount):
        new_text = tokenization(preprocessing(train_texts[i]))
        matr = np.append(matr, vectorization(new_text, embeddings), axis=0)

        if (train_labels[i] == 'neg'):
            y_labels = y_labels + [[0, 1]]
        else:
            y_labels = y_labels + [[1, 0]]

    matr = np.delete(matr, (0), axis=0)
    learning_rate = 0.01
    num_epochs = 1200
    batch_size = 5000
    layer_sizes = [d, 200, 100]
    dict_weights = train_ffnn(matr, np.array(y_labels), layer_sizes, learning_rate, num_epochs, batch_size)

    return {'weights': dict_weights, 'embeddings':embeddings}

def classify(texts: List[str], params: Any) -> List[str]:

    embeddings = params['embeddings']
    dict_weights = params['weights']

    doc_amount = len(texts) # amount of documents in train sample
    matr = np.ones((1, d))
    for i in range(doc_amount):
        new_text = tokenization(preprocessing(texts[i]))
        matr = np.append(matr, vectorization(new_text, embeddings), axis=0)

    matr = np.delete(matr, (0), axis=0)
    print("matr shape ", matr.shape)
    X, cache = ffnn(matr, dict_weights, "tanh")

    texts_labels = []
    print (X.shape)
    for i in range(X.shape[0]):
        if (X[i][0] > X[i][1]):
            texts_labels.append('pos')
        else:
            texts_labels.append('neg')
    return texts_labels


import cx_Oracle
import os
import pandas as pd
import geopy
from geopy.geocoders import Nominatim
from datetime import datetime, timedelta
import geocoder

os.environ["NLS_LANG"] = ".AL32UTF8"
location_list = ['경기', '강원', '경상남도', '경상북도', '충청남도', '충청북도', '전라남도', '전라북도', '제주특별자치도', '대전', '광주', '부산', '울산', '대구', '인천']

# sql = 'select LOCAL_ID from tempdata where local_id = ' + "'수지구'"
sql = 'select * from child_0 where 아동_성별 = ' + "'M'"

# cursor.execute(sql.encode('utf-8'))

def print_db():
    print("test1")
    cursor.execute(sql.encode('utf-8'))
    for row in cursor:
        print(row)
    print("test2")

# def read_database(case_num, child_id):
#     sql_read = "select * from child_0 where 개별사건번호 = '%d' and 피해아동대상자 = '%s'" %(int(case_num), child_id)
#     row = cursor.execute(sql_read.encode('utf-8'))
#     db_data = row.fetchone()
#     return db_data

    # locator = Nominatim(user_agent = "myGeocoder")
    # coordinates = "%f, %f" % (myloc.latlng[0], myloc.latlng[1])
    # location = locator.reverse(coordinates, language=RFC2616)
    # temp_province = location.raw['address']['province']

    # if()
def read_database():
    #DB 연결
    conn = cx_Oracle.connect("Jason/ssis1234@localhost:49161/xe")
    cursor = conn.cursor()

    #위치정보
    myloc = geocoder.ip('me')
    locator = Nominatim(user_agent = "myGeocoder")
    coordinates = "%s, %s" %(myloc.latlng[0], myloc.latlng[1])
    print(coordinates)
    location = locator.reverse(coordinates)
    my_location = None

    try:
        my_location = location.raw['address']['city']
    except(KeyError):
        pass
    try:
        my_location = location.raw['address']['province']
    except(KeyError):
        pass

    for i in range(len(location_list)):
        if my_location != location_list[i]:
            if (myloc.latlng[0] >= 37.413294 and myloc.latlng[0] <= 37.715133) and (myloc.latlng[1] >= 126.734086 and myloc.latlng[1] <= 127.269311):
                my_location = '서울'
            elif(myloc.latlng[0] >= 36.418608 and myloc.latlng[0] <= 36.733585) and (myloc.latlng[1] >= 127.126739 and myloc.latlng[1] <= 127.409310):
                my_location = '세종'
    if my_location == '경기도':
        my_location = '경기'
    elif my_location == '경상남도':
        my_location = '경남'
    elif my_location == '경상북도':
        my_location = '경북'
    elif my_location == '충청남도':
        my_location = '충남'
    elif my_location == '충청북도':
        my_location = '충북'
    elif my_location == '전라남도':
        my_location = '전남'
    elif my_location == '전라북도':
        my_location = '전북'
    elif my_location == '제주특별자치도':
        my_location = '제주'

    #날짜정보
    # case_num = request.form['case_num']
    # child_id = request.form['child_id']
    todayDate = datetime.now()
    # todayDate = tempDate.strftime('%Y-%m-%d')

    threeMonthBefore = todayDate - timedelta(days=820)
    sixMonthBefore = todayDate - timedelta(days=910)

    # 소문자 y -> 20, 대문자 Y -> 2020
    tMB_Date = threeMonthBefore.strftime('%y-%m-%d')
    sMB_Date = sixMonthBefore.strftime('%y-%m-%d')
    # db_data = db.read_database(case_num, child_id)

    tMB_Date = tMB_Date.replace('-', '/')
    sMB_Date = sMB_Date.replace('-', '/')

    # sql_read = """select * from child_0
    #             where (신고_통보일시 <= TO_DATE('%s', 'YY/MM/DD') and 신고_통보일시 >= TO_DATE('%s', 'YY/MM/DD')
    #             and 신대_통계거점 = '%s' and 결과_학대혐의여부 = 'Y')""" %(tMB_Date, sMB_Date, my_location)
    sql_read = """select 개별사건번호, 피해아동대상자, 학대행위자대상, 신대_통계거점, 결과_학대혐의여부 from child_0
                where (신고_통보일시 <= TO_DATE('%s', 'YY/MM/DD') and 신고_통보일시 >= TO_DATE('%s', 'YY/MM/DD')
                and 신대_통계거점 = '%s')""" %(tMB_Date, sMB_Date, my_location)
    row = cursor.execute(sql_read.encode('utf-8'))
    db_data = row.fetchall()
    conn.close()

    return db_data


from logging import getLogger

import numpy as np
from fusion_base import Fusion
from onnx import TensorProto, helper
from onnx_model import OnnxModel

logger = getLogger(__name__)

class FusionReshape(Fusion):
    def __init__(self, model: OnnxModel):
        super().__init__(model, "Reshape", "Reshape")
        self.prune_graph: bool = False

    def replace_reshape_node(self, shape, reshape_node, concat_node):
        shape_value = np.asarray(shape, dtype=np.int64)
        constant_shape_name = self.model.create_node_name("Constant", "constant_shape")
        new_node = helper.make_node(
            "Constant",
            inputs=[],
            outputs=[constant_shape_name],
            value=helper.make_tensor(
                name="const_tensor",
                data_type=TensorProto.INT64,
                dims=shape_value.shape,
                vals=bytes(shape_value),
                raw=True,
            ),
        )
        reshape_node.input[1] = constant_shape_name
        reshape_node.name = self.model.create_node_name("Reshape", "Reshape_Fuse")
        self.nodes_to_remove.extend([concat_node])
        self.nodes_to_add.append(new_node)
        self.node_name_to_graph_name[new_node.name] = self.this_graph_name

    def fuse(self, reshape_node, input_name_to_nodes, output_name_to_node):
        if reshape_node.input[1] not in output_name_to_node:
            return

        concat_node = output_name_to_node[reshape_node.input[1]]
        if concat_node.op_type != "Concat" or len(concat_node.input) < 3 or len(concat_node.input) > 4:
            return

        path0 = self.model.match_parent_path(
            concat_node,
            ["Unsqueeze", "Gather", "Shape"],
            [0, 0, 0],
            output_name_to_node,
        )
        if path0 is None:
            return

        (unsqueeze_0, gather_0, shape_0) = path0

        path1 = self.model.match_parent_path(
            concat_node,
            ["Unsqueeze", "Gather", "Shape"],
            [1, 0, 0],
            output_name_to_node,
        )
        if path1 is None:
            return
        (unsqueeze_1, gather_1, shape_1) = path1

        shape = []
        gather_value = self.model.get_constant_value(gather_0.input[1])
        if gather_value == 0:
            shape.append(0)

        gather_value = self.model.get_constant_value(gather_1.input[1])
        if gather_value == 1:
            shape.append(0)

        if len(shape) != 2:
            return

        path2 = []
        path3 = []
        shape_nodes = [shape_0, shape_1]
        if len(concat_node.input) == 3 and self.model.get_constant_value(concat_node.input[2]) is None:
            path2 = self.model.match_parent_path(
                concat_node,
                ["Unsqueeze", "Mul", "Gather", "Shape"],
                [2, 0, 0, 0],
                output_name_to_node,
            )
            if path2 is None:
                path2 = self.model.match_parent_path(
                    concat_node,
                    ["Unsqueeze", "Mul", "Squeeze", "Slice", "Shape"],
                    [2, 0, 0, 0, 0],
                    output_name_to_node,
                )  # GPT2 exported by PyTorch 1.4 with opset_version=11
                if path2 is None:
                    return

            path3 = self.model.match_parent_path(
                concat_node,
                ["Unsqueeze", "Mul", "Gather", "Shape"],
                [2, 0, 1, 0],
                output_name_to_node,
            )
            if path3 is None:
                path3 = self.model.match_parent_path(
                    concat_node,
                    ["Unsqueeze", "Mul", "Squeeze", "Slice", "Shape"],
                    [2, 0, 1, 0, 0],
                    output_name_to_node,
                )  # GPT2 exported by PyTorch 1.4 with opset_version=11
                if path3 is None:
                    return

            shape_nodes.extend([path2[-1], path3[-1]])
            shape.append(-1)
        elif len(concat_node.input) > 2:
            concat_value = self.model.get_constant_value(concat_node.input[2])
            if concat_value is None:
                return
            if isinstance(concat_value, np.ndarray):
                shape.extend(concat_value.tolist())
            else:
                shape.append(concat_value)

        if len(concat_node.input) == 4 and self.model.get_constant_value(concat_node.input[3]) is None:
            if -1 in shape:
                return

            path2 = self.model.match_parent_path(
                concat_node,
                ["Unsqueeze", "Div", "Gather", "Shape"],
                [3, 0, 0, 0],
                output_name_to_node,
            )
            if path2 is None:
                path2 = self.model.match_parent_path(
                    concat_node,
                    ["Unsqueeze", "Div", "Squeeze", "Slice", "Shape"],
                    [3, 0, 0, 0, 0],
                    output_name_to_node,
                )  # GPT2 exported by PyTorch 1.4 with opset_version=11
                if path2 is None:
                    return
            shape_nodes.extend([path2[-1]])
            shape.append(-1)
        elif len(concat_node.input) > 3:
            concat_value = self.model.get_constant_value(concat_node.input[3])
            if concat_value is None:
                return

            if isinstance(concat_value, np.ndarray):
                shape.extend(concat_value.tolist())
            else:
                shape.append(concat_value)

        root_input = reshape_node.input[0]
        same_shape_input = True
        for shape_node in shape_nodes:
            if shape_node.input[0] != root_input:
                same_shape_input = False

        if not same_shape_input:
            return

        self.replace_reshape_node(shape, reshape_node, concat_node)

        # TODO(tlwu): Subgraph blocks pruning un-used nodes. Add code to remove un-used nodes safely.
        self.prune_graph = True


from __future__ import print_function

import argparse
import json
import logging
import os
import pickle as pkl

import pandas as pd
import xgboost as xgb
from sagemaker_containers import entry_point
from sagemaker_xgboost_container import distributed
# from sagemaker_xgboost_container.data_utils import get_dmatrix
import time

import shutil
import csv

# Heavily draws from AWS's data_utils.py script:
# https://github.com/aws/sagemaker-xgboost-container/blob/cf2a05a525606225e91d7e588f57143827cbe3f7/src/sagemaker_xgboost_container/data_utils.py
# but we use our own version because the AWS version does not allow you to specify the label column.
def get_dmatrix(data_path, content_type, label_column, csv_weights=0):
    """Create Data Matrix from CSV file.
    Assumes that sanity validation for content type has been done.
    :param data_path: Either directory or file
    :param content_type: Only supports "csv"
    :param label_column: Integer corresponding to index of the label column, starting with 0
    :param csv_weights: 1 if the instance weights are in the second column of csv file; otherwise, 0
    :return: xgb.DMatrix or None
    """

    if "csv" not in content_type.lower():
        raise Exception("File type '{}' not supported".format(content_type))

    if not isinstance(data_path, list):
        if not os.path.exists(data_path):
            logging.info('File path {} does not exist!'.format(data_path))
            return None
        files_path = get_files_path(data_path)
    else:
        # Create a directory with symlinks to input files.
        files_path = "/tmp/sagemaker_xgboost_input_data"
        shutil.rmtree(files_path, ignore_errors=True)
        os.mkdir(files_path)
        for path in data_path:
            if not os.path.exists(path):
                return None
            if os.path.isfile(path):
                os.symlink(path, os.path.join(files_path, os.path.basename(path)))
            else:
                for file in os.scandir(path):
                    os.symlink(file, os.path.join(files_path, file.name))

    dmatrix = get_csv_dmatrix(files_path, label_column, csv_weights)

    return dmatrix

def get_files_path(data_path):
    if os.path.isfile(data_path):
        files_path = data_path
    else:
        for root, dirs, files in os.walk(data_path):
            if dirs == []:
                files_path = root
                break

    return files_path

def get_csv_dmatrix(files_path, label_column, csv_weights):
    """Get Data Matrix from CSV data in file mode.
    Infer the delimiter of data from first line of first data file.
    :param files_path: File path where CSV formatted training data resides, either directory or file
    :param label_column: Integer corresponding to index of the label column, starting with 0
    :param csv_weights: 1 if instance weights are in second column of CSV data; else 0
    :return: xgb.DMatrix
    """
    csv_file = files_path if os.path.isfile(files_path) else [
        f for f in os.listdir(files_path) if os.path.isfile(os.path.join(files_path, f))][0]
    with open(os.path.join(files_path, csv_file)) as read_file:
        sample_csv_line = read_file.readline()
    delimiter = _get_csv_delimiter(sample_csv_line)

    try:
        if csv_weights == 1:
            dmatrix = xgb.DMatrix(
                '{}?format=csv&label_column={}&delimiter={}&weight_column=1'.format(files_path, label_column, delimiter))
        else:
            dmatrix = xgb.DMatrix('{}?format=csv&label_column={}&delimiter={}'.format(files_path, label_column, delimiter))

    except Exception as e:
        raise Exception("Failed to load csv data with exception:\n{}".format(e))

    return dmatrix

def _get_csv_delimiter(sample_csv_line):
    try:
        delimiter = csv.Sniffer().sniff(sample_csv_line).delimiter
        logging.info("Determined delimiter of CSV input is \'{}\'".format(delimiter))
    except Exception as e:
        raise Exception("Could not determine delimiter on line {}:\n{}".format(sample_csv_line[:50], e))
    return delimiter

def _xgb_train(params, dtrain, evals, num_boost_round, model_dir, is_master):
    """Run xgb train on arguments given with rabit initialized.

    This is our rabit execution function.

    :param args_dict: Argument dictionary used to run xgb.train().
    :param is_master: True if current node is master host in distributed training,
                        or is running single node training job.
                        Note that rabit_run will include this argument.
    """

    start = time.time()
    booster = xgb.train(params=params, dtrain=dtrain, evals=evals, num_boost_round=num_boost_round)
    logging.info("XGBoost training time {}".format(time.time() - start))

    if is_master:
        model_location = model_dir + "/xgboost-model"
        pkl.dump(booster, open(model_location, "wb"))
        logging.info("Stored trained model at {}".format(model_location))

if __name__ == "__main__":
    parser = argparse.ArgumentParser()

    # Hyperparameters are described here.
    parser.add_argument(
        "--max_depth",
        type=int,
    )
    parser.add_argument("--eta", type=float)
    parser.add_argument("--gamma", type=int)
    parser.add_argument("--min_child_weight", type=int)
    parser.add_argument("--subsample", type=float)
    parser.add_argument("--verbosity", type=int)
    parser.add_argument("--objective", type=str)
    parser.add_argument("--num_round", type=int)
    parser.add_argument("--tree_method", type=str, default="gpu_hist")  # "auto", "hist", or "gpu_hist"
    parser.add_argument("--predictor", type=str, default="gpu_predictor")  # "auto"

    # e.g., 'sklearn.datasets.fetch_california_housing()'
    parser.add_argument("--sklearn_dataset", type=str, default="None")
    # specify file type
    parser.add_argument("--content_type", type=str)
    # if csv, should specify a label column
    parser.add_argument("--label_column", type=int)

    # Sagemaker specific arguments. Defaults are set in the environment variables.
    parser.add_argument("--output_data_dir", type=str, default=os.environ.get("SM_OUTPUT_DATA_DIR"))
    parser.add_argument("--model_dir", type=str, default=os.environ.get("SM_MODEL_DIR"))
    parser.add_argument("--train", type=str, default=os.environ.get("SM_CHANNEL_TRAIN"))
    parser.add_argument("--validation", type=str, default=os.environ.get("SM_CHANNEL_VALIDATION"))
    parser.add_argument("--sm_hosts", type=str, default=os.environ.get("SM_HOSTS"))
    parser.add_argument("--sm_current_host", type=str, default=os.environ.get("SM_CURRENT_HOST"))

    args, _ = parser.parse_known_args()

    # Get SageMaker host information from runtime environment variables
    sm_hosts = json.loads(args.sm_hosts)
    sm_current_host = args.sm_current_host

    sklearn_dataset = args.sklearn_dataset
    if "None" in sklearn_dataset:
        dtrain = get_dmatrix(args.train, args.content_type, args.label_column)
        try:
            dval = get_dmatrix(args.validation, args.content_type, args.label_column)
        except Exception:
            dval = None
    else:  # Use a dataset from sklearn.datasets
        import sklearn.datasets
        try:
            # e.g., sklearn_dataset = "sklearn.datasets.fetch_california_housing()"
            data = eval(sklearn_dataset)
        except Exception:
            raise ValueError("Function {} is not supported. Try something like 'sklearn.datasets.fetch_california_housing()'"
                             .format(sklearn_dataset))

        X = data.data
        y = data.target
        dtrain = xgb.DMatrix(X, y)
        dval = None

    watchlist = (
        [(dtrain, "train"), (dval, "validation")] if dval is not None else [(dtrain, "train")]
    )

    train_hp = {
        "max_depth": args.max_depth,
        "eta": args.eta,
        "gamma": args.gamma,
        "min_child_weight": args.min_child_weight,
        "subsample": args.subsample,
        "verbosity": args.verbosity,
        "objective": args.objective,
        "tree_method": args.tree_method,
        "predictor": args.predictor,
    }

    xgb_train_args = dict(
        params=train_hp,
        dtrain=dtrain,
        evals=watchlist,
        num_boost_round=args.num_round,
        model_dir=args.model_dir,
    )

    if len(sm_hosts) > 1:
        # Wait until all hosts are able to find each other
        entry_point._wait_hostname_resolution()

        # Execute training function after initializing rabit.
        distributed.rabit_run(
            exec_fun=_xgb_train,
            args=xgb_train_args,
            include_in_training=(dtrain is not None),
            hosts=sm_hosts,
            current_host=sm_current_host,
            update_rabit_args=True,
        )
    else:
        # If single node training, call training method directly.
        if dtrain:
            xgb_train_args["is_master"] = True
            _xgb_train(**xgb_train_args)
        else:
            raise ValueError("Training channel must have data to train model.")

def model_fn(model_dir):
    """Deserialize and return fitted model.

    Note that this should have the same name as the serialized model in the _xgb_train method
    """
    model_file = "xgboost-model"
    booster = pkl.load(open(os.path.join(model_dir, model_file), "rb"))
    return booster