## User Agent: 유저가 사람임을 인식 시키는 코드
## User Agent 가 없으면 웹 사이트에서 침입으로 인색해 차단할 가능성이 있음

# User Agent string을 검색해서 자신의 user agent를 가져옴
import requests
url = 'http://nadocoding.tistory.com'

# user agent를 입력
headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.146 Safari/537.36'}

# requests.get에 url뿐 아니라 user agent를 넣어줘서 실제 검색할 때와 같은 조건으로 웹 스크래핑을 해옴
res = requests.get(url, headers = headers)
res.raise_for_status()

with open('requests_test2.html', 'w', encoding='utf8') as f:
    f.write(res.text)


# -*- coding: utf-8 -*-
"""
Created on Fri Jun  8 11:32:53 2018

@author: KRUEGKJ

logreg_model_investigation.py
"""

import sys
sys.path.append('../lib')
sys.path.append('../utilities')

from plot_utils import *
from retrieve_data import *
from indicators import *
from transformers import *
#from stat_tests import *

# Import the Time Series library
#import statsmodels.tsa.stattools as ts
import pandas as pd
import numpy as np
import datetime
from dateutil.relativedelta import relativedelta
import matplotlib.pylab as plt
#plt.style.use('seaborn-ticks')
from pandas.tseries.holiday import USFederalHolidayCalendar
from pandas.tseries.offsets import CustomBusinessDay
from pandas.tseries.offsets import BDay
#import matplotlib as mpl
#plt.style.use('seaborn-ticks')
#import matplotlib.ticker as ticker

from sklearn.model_selection import StratifiedShuffleSplit
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import confusion_matrix
from sklearn.metrics import accuracy_score
from sklearn.metrics import precision_score
from sklearn.metrics import recall_score
from sklearn.metrics import f1_score

import os.path

issue = "XLE"

us_cal = CustomBusinessDay(calendar=USFederalHolidayCalendar())
pivotDate = datetime.date(2018, 2, 2)
inSampleOutOfSampleRatio = 2
outOfSampleMonths = 12
inSampleMonths = inSampleOutOfSampleRatio * outOfSampleMonths
print("inSampleMonths: " + str(inSampleMonths))
segments = 1
months_to_load = outOfSampleMonths + segments * inSampleMonths
print("Months to load: " + str(months_to_load))

inSampleStartDate = pivotDate - relativedelta(months=months_to_load)
dataLoadStartDate = inSampleStartDate - relativedelta(months=1)
ooSampleStartDate = pivotDate - relativedelta(months=outOfSampleMonths)
print("Load Date: ", dataLoadStartDate)
print("In Sample Start  Date: ", inSampleStartDate)
print("Out Of Sample Start  Date: ", ooSampleStartDate)
print("Pivot Date: ", pivotDate)

dSet = DataRetrieve()
dataSet = dSet.read_issue_data(issue)
dataSet = dSet.set_date_range(dataSet, dataLoadStartDate,pivotDate)

#dataSet = read_issue_data(issue, dataLoadStartDate, pivotDate)
print(issue)
nrows = dataSet.shape[0]
print ("nrows: ", nrows)

addIndic1 = Indicators()
ind_list = [("RSI", 1.5),("ROC",2),("DPO",3),("ATR", 3)]
dataSet = addIndic1.add_indicators(dataSet, ind_list)

zScore_lookback = 3
transf = Transformers()
transfList = ['Pri_ROC','Pri_DPO','Pri_ATR']
for i in transfList:
    dataSet = transf.zScore_transform(dataSet, zScore_lookback, i)

lags = 2
transfList = ['Pri_ROC_zScore','Pri_DPO_zScore','Pri_ATR_zScore']
for i in transfList:
    dataSet = transf.add_lag(dataSet, i, lags)

# set lag on Close (Pri)
transf = Transformers()
lag_var = 'Pri'
lags = 6
dataSet = transf.add_lag(dataSet, lag_var, lags)

# TURN THIS INTO A FUNCTION
# add Close Higher features
dataSet['1DayHigherClose'] = dataSet['Pri'] > dataSet['Pri_lag1']
dataSet['2DayHigherClose'] = dataSet['Pri'] > dataSet['Pri_lag2']
dataSet['3DayHigherClose'] = dataSet['Pri'] > dataSet['Pri_lag3']
dataSet['4DayHigherClose'] = dataSet['Pri'] > dataSet['Pri_lag4']

dataSet['1DayLowerClose'] = dataSet['Pri'] < dataSet['Pri_lag1']
dataSet['2DayLowerClose'] = dataSet['Pri'] < dataSet['Pri_lag2']
dataSet['3DayLowerClose'] = dataSet['Pri'] < dataSet['Pri_lag3']
dataSet['4DayLowerClose'] = dataSet['Pri'] < dataSet['Pri_lag4']

# set % return variables and lags
dataSet["percReturn"] = dataSet["Pri"].pct_change()*100
lag_var = 'percReturn'
lags = 4
dataSet = transf.add_lag(dataSet, lag_var, lags)

predictor_vars = 'Price and percReturn lags'
# Put indicators and transforms here

#set beLong level
beLongThreshold = 0
ct = ComputeTarget()
dataSet = ct.setTarget(dataSet, "Long", beLongThreshold)

modelStartDate = inSampleStartDate
modelEndDate = modelStartDate + relativedelta(months=inSampleMonths)

plotIt = PlotUtility()

df2 = pd.date_range(start=modelStartDate, end=modelEndDate, freq=us_cal)
mmData = dataSet.reindex(df2)

nrows = mmData.shape[0]
print ("beLong counts: ")
be_long_count = mmData['beLong'].value_counts()
print (be_long_count)
print ("out of ", nrows)

mmData = mmData.drop(['Open','High','Low','Close', 'percReturn', 'gainAhead', 'Symbol','Pri_ROC','Pri_DPO','Pri_ATR'],axis=1)

plotTitle = issue + ", " + str(modelStartDate) + " to " + str(modelEndDate)
plotIt.plot_v2x(mmData['Pri'], mmData['beLong'], plotTitle)
plotIt.histogram(mmData['beLong'], x_label="beLong signal", y_label="Frequency",
  title = "beLong distribution for " + issue)
plt.show(block=False)

mmData = mmData.drop(['Pri'],axis=1)

# Chnage -1 to 0 for log reg
#mmData.beLong.replace([1, -1], [1, 0], inplace=True)
datay = mmData['beLong']
nrows = datay.shape[0]
print ("nrows beLong: ", nrows)

mmData = mmData.drop(['beLong'],axis=1)
dataX = mmData

# get names of features
names = dataX.columns.values.tolist()

"""
# correlation plot
def plot_corr(df,size=10):
    '''Function plots a graphical correlation matrix for each pair of columns in the dataframe.

    Input:
        df: pandas DataFrame
        size: vertical and horizontal size of the plot'''

    corr = df.corr()
    fig, ax = plt.subplots(figsize=(size, size))
    ax.matshow(corr)
    plt.xticks(range(len(corr.columns)), corr.columns, rotation='vertical');
    plt.yticks(range(len(corr.columns)), corr.columns);
plot_corr(dataX,8)
"""

#  Copy from pandas dataframe to numpy arrays
dy = np.zeros_like(datay)
dX = np.zeros_like(dataX)

dy = datay.values
dX = dataX.values

#########################################
#import statsmodels.api as sm
#logit_model=sm.Logit(datay,dataX)
#result=logit_model.fit()
#print(result.summary())
#####################################

######################
# ML section
model_results = []

iterations = 100

model = RandomForestClassifier(n_jobs=-1, random_state=0, min_samples_split=20, n_estimators=500, max_features = 'auto', min_samples_leaf = 20, oob_score = 'TRUE')
modelname = 'RF'

#  Make 'iterations' index vectors for the train-test split
sss = StratifiedShuffleSplit(n_splits=iterations,test_size=0.33, random_state=None)

accuracy_scores_is = []
accuracy_scores_oos = []
precision_scores_is = []
precision_scores_oos = []
recall_scores_is = []
recall_scores_oos = []
f1_scores_is = []
f1_scores_oos = []

#  Initialize the confusion matrix
cm_sum_is = np.zeros((2,2))
cm_sum_oos = np.zeros((2,2))

#  For each entry in the set of splits, fit and predict
for train_index,test_index in sss.split(dX,dy):
    X_train, X_test = dX[train_index], dX[test_index]
    y_train, y_test = dy[train_index], dy[test_index]

#  fit the model to the in-sample data
    model.fit(X_train, y_train)

#  test the in-sample fit
    y_pred_is = model.predict(X_train)
    #for i in range(0, 5):
    #    print ("Actual outcome :: {} and Predicted outcome :: {}".format(list(y_train)[i], y_pred_is[i]))
    cm_is = confusion_matrix(y_train, y_pred_is)
    cm_sum_is = cm_sum_is + cm_is
    accuracy_scores_is.append(accuracy_score(y_train, y_pred_is))
    precision_scores_is.append(precision_score(y_train, y_pred_is))
    recall_scores_is.append(recall_score(y_train, y_pred_is))
    f1_scores_is.append(f1_score(y_train, y_pred_is))

#  test the out-of-sample data
    y_pred_oos = model.predict(X_test)
    #for i in range(0, 5):
    #    print ("Actual outcome :: {} and Predicted outcome :: {}".format(list(y_test)[i], y_pred_oos[i]))
    cm_oos = confusion_matrix(y_test, y_pred_oos)
    cm_sum_oos = cm_sum_oos + cm_oos
    accuracy_scores_oos.append(accuracy_score(y_test, y_pred_oos))
    precision_scores_oos.append(precision_score(y_test, y_pred_oos))
    recall_scores_oos.append(recall_score(y_test, y_pred_oos))
    f1_scores_oos.append(f1_score(y_test, y_pred_oos))

print (sorted(zip(map(lambda x: round(x, 2), model.feature_importances_), names), reverse=True))

tpIS = cm_sum_is[1,1]
fnIS = cm_sum_is[1,0]
fpIS = cm_sum_is[0,1]
tnIS = cm_sum_is[0,0]
precisionIS = tpIS/(tpIS+fpIS)
recallIS = tpIS/(tpIS+fnIS)
accuracyIS = (tpIS+tnIS)/(tpIS+fnIS+fpIS+tnIS)
f1IS = (2.0 * precisionIS * recallIS) / (precisionIS+recallIS)

tpOOS = cm_sum_oos[1,1]
fnOOS = cm_sum_oos[1,0]
fpOOS = cm_sum_oos[0,1]
tnOOS = cm_sum_oos[0,0]
precisionOOS = tpOOS/(tpOOS+fpOOS)
recallOOS = tpOOS/(tpOOS+fnOOS)
accuracyOOS = (tpOOS+tnOOS)/(tpOOS+fnOOS+fpOOS+tnOOS)
f1OOS = (2.0 * precisionOOS * recallOOS) / (precisionOOS+recallOOS)

print ("\n\nSymbol is ", issue)
print ("Learning algorithm is ", model)
print ("Confusion matrix for %i randomized tests" % iterations)
print ("for dates ", modelStartDate, " through ", modelEndDate)
print ("----------------------")
print ("==In Sample==")
print ("     Predicted")
print ("      pos neg")
print ("pos:  %i  %i  Recall:%.2f" % (tpIS, fnIS, recallIS))
print ("neg:  %i  %i" % (fpIS, tnIS))
print ("Prec: %.2f          Accuracy: %.2f " % (precisionIS, accuracyIS))
print ("f1:   %.2f" % f1IS)
print ("\n----------------------")
print ("\n==Out Of Sample==")
print ("     Predicted")
print ("      pos neg")
print ("pos:  %i  %i  Recall:%.2f" % (tpOOS, fnOOS, recallOOS))
print ("neg:  %i  %i" % (fpOOS, tnOOS))
print ("Prec: %.2f          Accuracy: %.2f " % (precisionOOS, accuracyOOS))
print ("f1:   %.2f" % f1OOS)
print ("\n----------------------")

print ("\n==In Sample==")
print('Accuracy: %.2f' % np.mean(accuracy_scores_is))
print('Precision: %.2f' % np.mean(precision_scores_is))
print('Recall: %.2f' % np.mean(recall_scores_is))
print('F1: %.2f' % np.mean(f1_scores_is))
print ("\n==Out Of Sample==")
print('Accuracy: %.2f' % np.mean(accuracy_scores_oos))
print('Precision: %.2f' % np.mean(precision_scores_oos))
print('Recall: %.2f' % np.mean(recall_scores_oos))
print('F1: %.2f' % np.mean(f1_scores_oos))
print ("\nend of run")

model_results.append({'Issue': issue, 'StartDate': modelStartDate.strftime("%Y-%m-%d"), 'EndDate': modelEndDate.strftime("%Y-%m-%d"), 'Model': modelname, 'Rows': nrows, 'beLongCount': str(np.count_nonzero(dy==1)), 'Predictors': predictor_vars, 'IS-Accuracy': np.mean(accuracy_scores_is), 'IS-Precision': np.mean(precision_scores_is), 'IS-Recall': np.mean(recall_scores_is), 'IS-F1': np.mean(f1_scores_is), 'OOS-Accuracy':  np.mean(accuracy_scores_oos), 'OOS-Precision': np.mean(precision_scores_oos), 'OOS-Recall': np.mean(recall_scores_oos), 'OOS-F1': np.mean(f1_scores_oos)})

df = pd.DataFrame(model_results)
df = df[['Issue','StartDate','EndDate','Model','Rows','beLongCount','Predictors','IS-Accuracy','IS-Precision','IS-Recall','IS-F1','OOS-Accuracy','OOS-Precision','OOS-Recall','OOS-F1']]
print(df)

dirext = issue + '_Model_' + modelname + '_start_' + str(dataLoadStartDate) + '_end_' + str(pivotDate) + '_TS_' + datetime.datetime.now().strftime("%Y-%m-%d")
print(dirext)
filename = "IS_model_iteration_" + dirext + ".csv"
current_directory = os.getcwd()
df.to_csv(current_directory+"\\"+filename, encoding='utf-8', index=False)

"""
Adding validation code
"""
# Select the date range
modelStartDate = ooSampleStartDate
modelEndDate = modelStartDate + relativedelta(months=outOfSampleMonths)

df2 = pd.date_range(start=modelStartDate, end=modelEndDate, freq=us_cal)
valData = dataSet.reindex(df2)
tradesData = valData

nrows = valData.shape[0]
print ("beLong counts: ")
be_long_count = valData['beLong'].value_counts()
print (be_long_count)
print ("out of ", nrows)

valData = valData.drop(['Open','High','Low','Close', 'percReturn', 'Symbol','Pri_ROC','Pri_DPO','Pri_ATR'],axis=1)

plotTitle = issue + ", " + str(modelStartDate) + " to " + str(modelEndDate)
plotIt.plot_v2x(valData['Pri'], valData['beLong'], plotTitle)
plotIt.histogram(valData['beLong'], x_label="beLong signal", y_label="Frequency",
  title = "beLong distribution for " + issue)
plt.show(block=False)

valModelData = valData.drop(['Pri','beLong','gainAhead'],axis=1)

valRows = valModelData.shape[0]
print("There are %i data points" % valRows)

# test the validation data
y_validate = []
y_validate = model.predict(valModelData)

# Create best estimate of trades
bestEstimate = np.zeros(valRows)

# You may need to adjust for the first and / or final entry
for i in range(valRows -1):
    print(valData.gainAhead.iloc[i], y_validate[i])
    if y_validate[i] > 0.0:
        bestEstimate[i] = valData.gainAhead.iloc[i]
    else:
        bestEstimate[i] = 0.0

# Create and plot equity curve
equity = np.zeros(valRows)
equity[0] = 1.0
for i in range(1,valRows):
    equity[i] = (1+bestEstimate[i])*equity[i-1]

print("\nTerminal Weatlh: ", equity[valRows-1])
plt.plot(equity)

print("\n End of Run")

valData['valBeLong'] = pd.Series(y_validate, index=valData.index)

#==========================
import matplotlib as mpl
#from matplotlib import style
plt.style.use('seaborn-ticks')
import matplotlib.ticker as ticker

fig = plt.figure(figsize=(12,8))
plt.subplots_adjust(hspace=0.05)
ax1 = plt.subplot2grid((6,1), (0,0), rowspan=4, colspan=1)
ax2 = plt.subplot2grid((6,1), (4,0), rowspan=1, colspan=1)
ax3 = plt.subplot2grid((6,1), (5,0), rowspan=1, colspan=1)

ax2.plot(valData['valBeLong'], color='green', alpha =0.6)
ax1.plot(valData['Pri'])
ax3.plot(valData['beLong'], color='purple', alpha =0.6)

ax1.label_outer()
ax2.label_outer()
ax2.tick_params(axis='x',which='major',bottom='on')
ax1.grid(True, which='major', color='k', linestyle='-', alpha=0.6)
ax2.grid(True, which='major', color='k', linestyle='-', alpha=0.6)
ax3.grid(True, which='major', color='k', linestyle='-', alpha=0.6)
ax1.grid(True, which='minor', color='r', linestyle='-', alpha=0.2)
ax2.grid(True, which='minor', color='r', linestyle='-', alpha=0.2)
ax3.grid(True, which='minor', color='r', linestyle='-', alpha=0.2)
ax1.minorticks_on()
ax2.minorticks_on()
ax3.minorticks_on()
ax1.label_outer()
ax1.legend(loc='upper left', frameon=True, fontsize=8)
ax2.label_outer()
ax2.legend(loc='upper left', frameon=True, fontsize=8)
ax3.label_outer()
ax3.legend(loc='upper left', frameon=True, fontsize=8)

#==========================
tradesData['valBeLong'] = pd.Series(y_validate, index=tradesData.index)
tradesData['gain'] = tradesData['Close'] - tradesData['Open']

#  Count the number of rows in the file
nrows = tradesData.shape[0]
print ('There are %0.f rows of data' % nrows)

#  Compute cumulative equity for all days
equityAllSignals = np.zeros(nrows)
equityAllSignals[0] = 1
for i in range(1,nrows):
    equityAllSignals[i] = (1+tradesData.gainAhead[i])*equityAllSignals[i-1]

print ('TWR for all signals is %0.3f' % equityAllSignals[nrows-1])
# add to valData
valData['equityAllSignals'] = pd.Series(equityAllSignals, index=valData.index)

#  Compute cumulative equity for days with beLong signals
equityBeLongSignals = np.zeros(nrows)
equityBeLongSignals[0] = 1
for i in range(1,nrows):
    if (tradesData.beLong[i] > 0):
        equityBeLongSignals[i] = (1+tradesData.gainAhead[i])*equityBeLongSignals[i-1]
    else:
        equityBeLongSignals[i] = equityBeLongSignals[i-1]
valData['equityBeLongSignals'] = pd.Series(equityBeLongSignals, index=valData.index)

#  Compute cumulative equity for days with Validation beLong signals
equityValBeLongSignals = np.zeros(nrows)
equityValBeLongSignals[0] = 1
for i in range(1,nrows):
    if (tradesData.valBeLong[i] > 0):
        equityValBeLongSignals[i] = (1+tradesData.gainAhead[i])*equityValBeLongSignals[i-1]
    else:
        equityValBeLongSignals[i] = equityValBeLongSignals[i-1]

print ('TWR for all days with beLong signals is %0.3f' % equityBeLongSignals[nrows-1])
valData['equityValBeLongSignals'] = pd.Series(equityValBeLongSignals, index=valData.index)

#  Plot the two equity streams
fig = plt.figure(figsize=(12,8))
fig.suptitle(issue + ' Portfolio value in Validation')
ax1 = fig.add_subplot(111)
ax1.plot(valData.equityBeLongSignals, color='green',label='BeLong')
ax1.plot(valData.equityAllSignals, color='blue',label='Equity')
ax1.plot(valData.equityValBeLongSignals, color='purple',label='ValBeLong')

ax1.legend(loc='upper left', frameon=True, fontsize=8)
ax1.label_outer()
ax1.tick_params(axis='x',which='major',bottom='on')
ax1.minorticks_on()
ax1.grid(True, which='major', color='k', linestyle='-', alpha=0.6)
ax1.grid(True, which='minor', color='r', linestyle='-', alpha=0.2)

ax2 = ax1.twinx()
ax2.plot(valData.Pri, color='black',alpha=0.6,label='CLOSE',linestyle='--')
ax2.legend(loc='center left', frameon=True, fontsize=8)
ax2.label_outer()
plt.show()

#===================================
# Getting sample equity curve

#  Evaluation of signals

print ("Starting single run")

#  Status variables

ndays = tradesData.shape[0]

#  Local variables for trading system

initialEquity = 100000
fixedTradeDollars = 10000
commission = 0.005      #  Dollars per share per trade

#  These are scalar and apply to the current conditions

entryPrice = 0
exitPrice = 0

#  These have an element for each day loaded
#  Some will be unnecessary

accountBalance = np.zeros(ndays)
cash = np.zeros(ndays)
sharesHeld = np.zeros(ndays)
tradeGain = []
tradeGainDollars = []
openTradeEquity = np.zeros(ndays)
tradeWinsValue = np.zeros(ndays)
tradeLossesValue = np.zeros(ndays)

iTradeDay = 0
iTradeNumber = 0

#  Day 0 contains the initial values
accountBalance[0] = initialEquity
cash[0] = accountBalance[0]
sharesHeld[0] = 0

#  Loop over all the days loaded
for i in range (1,ndays):
    #  Extract the date
    dt = tradesData.index[i]
    #  Check the date
    datesPass = dt.date()>=modelStartDate and dt.date()<=modelEndDate
    if datesPass:
        iTradeDay = iTradeDay + 1
        if sharesHeld[iTradeDay-1] > 0:
            #  In a long position
            if tradesData.valBeLong[i]<0:
                #  target is -1 -- beFlat
                #  Exit -- close the trade
                exitPrice = tradesData.Pri[i]
                grossProceeds = sharesHeld[iTradeDay-1] * exitPrice
                commissionAmount = sharesHeld[iTradeDay-1] * commission
                netProceeds = grossProceeds - commissionAmount
                #print("netProceeds: ", netProceeds)
                cash[iTradeDay] = cash[iTradeDay-1] + netProceeds
                accountBalance[iTradeDay] = cash[iTradeDay]
                sharesHeld[iTradeDay] = 0
                iTradeNumber = iTradeNumber+1
                #tradeGain[iTradeNumber] = (exitPrice / (1.0 * entryPrice))
                tradeGain.append(exitPrice / (1.0 * entryPrice))
                tradeGainDollars.append(((exitPrice / (1.0 * entryPrice))*fixedTradeDollars)-fixedTradeDollars)

                pass
            else:
                #  target is +1 -- beLong
                #  Continue long
                sharesHeld[iTradeDay] = sharesHeld[iTradeDay-1]
                cash[iTradeDay] = cash[iTradeDay-1]
                MTMPrice = tradesData.Pri[i]
                openTradeEquity = sharesHeld[iTradeDay] * MTMPrice
                accountBalance[iTradeDay] = cash[iTradeDay] + openTradeEquity
                pass
        else:
            #  Currently flat
            if tradesData.valBeLong[i]>0:
                #  target is +1 -- beLong
                #  Enter a new position
                entryPrice = tradesData.Pri[i]
                sharesHeld[iTradeDay] = int(fixedTradeDollars/(entryPrice+commission))
                shareCost = sharesHeld[iTradeDay]*(entryPrice+commission)
                cash[iTradeDay] = cash[iTradeDay-1] - shareCost
                openTradeEquity = sharesHeld[iTradeDay]*entryPrice
                accountBalance[iTradeDay] = cash[iTradeDay] + openTradeEquity
                pass
            else:
                #  target is -1 -- beFlat
                #  Continue flat
                cash[iTradeDay] = cash[iTradeDay-1]
                accountBalance[iTradeDay] = cash[iTradeDay]
                pass

#  Format and print results

finalAccountBalance = accountBalance[iTradeDay]
print ('Final account balance: %.2f' %  finalAccountBalance)
numberTradeDays = iTradeDay
numberTrades = iTradeNumber
print ("Number of trades:", numberTrades)

from pandas import Series

Sequity = Series(accountBalance[0:numberTradeDays-1])

Sequity.plot()

tradeWins = sum(1 for x in tradeGain if float(x) >= 1.0)
tradeLosses = sum(1 for x in tradeGain if float(x) < 1.0 and float(x) > 0)
print("Wins: ", tradeWins)
print("Losses: ", tradeLosses)
print("W/L: ", tradeWins/numberTrades)

tradeWinsValue = sum((x*fixedTradeDollars)-fixedTradeDollars for x in tradeGain if float(x) >= 1.0)
tradeLossesValue = sum((x*fixedTradeDollars)-fixedTradeDollars for x in tradeGain if float(x) < 1.0 and float(x) > 0)
print('Total value of Wins:  %.2f' % tradeWinsValue)
print('Total value of Losses:  %.2f' % tradeLossesValue)
#(Win % x Average Win Size) – (Loss % x Average Loss Size)
print('Expectancy:  %.2f' % ((tradeWins/numberTrades)*(tradeWinsValue/tradeWins)-(tradeLosses/numberTrades)*(tradeLossesValue/tradeLosses)))
print("Fixed trade size: ", fixedTradeDollars)

# Sharpe ratio...probably not correct math
#import math
#print(np.mean(tradeGainDollars))
#print(np.std(tradeGainDollars))
#print(math.sqrt(numberTradeDays)*(np.mean(tradeGainDollars)/np.std(tradeGainDollars)))

####  end  ####
df_to_save = tradesData[['valBeLong','gainAhead']].copy()
df_to_save.reset_index(level=df_to_save.index.names, inplace=True)
df_to_save.columns=['Date','signal','gainAhead']
print(df_to_save)

dirext = issue + '_test1'
filename = "oos_equity_eval_" + dirext + ".csv"
df_to_save.to_csv(filename, encoding='utf-8', index=False)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models
from django.conf import settings

class Migration(migrations.Migration):

    dependencies = [
        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
        ('login', '0002_appt'),
    ]

    operations = [
        migrations.RenameField(
            model_name='appt',
            old_name='doctor_name',
            new_name='prescription',
        ),
        migrations.RemoveField(
            model_name='appt',
            name='patient_user',
        ),
        migrations.RemoveField(
            model_name='appt',
            name='uname',
        ),
        migrations.AddField(
            model_name='appt',
            name='doctor',
            field=models.ForeignKey(related_name='doctor_user', to=settings.AUTH_USER_MODEL, null=True),
        ),
        migrations.AddField(
            model_name='appt',
            name='patient',
            field=models.ForeignKey(related_name='patient_user', to=settings.AUTH_USER_MODEL, null=True),
        ),
    ]


# Bing Image Search API v7 - Image Scraper Application
# Adapted from Adrian Rosebrock's Bing image scraper
# Source: https://www.pyimagesearch.com/2018/04/09/how-to-quickly-build-a-deep-learning-image-dataset/

# Python libraries
import os
import logging
import argparse
import time
# 3rd Party Libraries
import cv2
import numpy as np
import requests
# User libraries
from keys.azure import subscription_key

logger = logging.getLogger()
logger.setLevel(logging.INFO)

ap = argparse.ArgumentParser()
ap.add_argument("-q", "--query", required=True,
    help="search query to search Bing Image API for")
ap.add_argument("-o", "--output", required=False,
    help="path to output directory of images")
ap.add_argument("-m", "--max", required=False,
    help="maximum number of search results")
ap.add_argument("-g", "--group", required=False,
    help="group size to process query")
ap.add_argument("-t", "--timeout", required=False,
    help="timeout in seconds to wait for an image to download")
ap.add_argument("--width", required=False,
    help="minimum required width")
ap.add_argument("--height", required=False,
    help="minimum required height")
ap.add_argument("-y", "--yolo", required=True,
    help="base path to YOLO directory")
ap.add_argument("--confidence", type=float, default=0.5,
    help="minimum probability to filter weak detections")
ap.add_argument("--threshold", type=float, default=0.3,
    help="threshold when applyong non-maxima suppression")
args = vars(ap.parse_args())

"""
# Load the image classifier
"""
# load the COCO class labels our YOLO model was trained on
labelsPath = os.path.sep.join([args["yolo"], "coco.names"])
LABELS = open(labelsPath).read().strip().split("\n")

# initialize a list of colors to represent each possible class label
np.random.seed(int(time.time()))
COLORS = np.random.randint(0, 255, size=(len(LABELS), 3),
                           dtype="uint8")

# derive the paths to the YOLO weights and model configuration
weightsPath = os.path.sep.join([args["yolo"], "yolov3.weights"])
configPath = os.path.sep.join([args["yolo"], "yolov3.cfg"])

# load our YOLO object detector trained on COCO dataset (80 classes)
print("[INFO] loading YOLO from disk...")
net = cv2.dnn.readNetFromDarknet(configPath, weightsPath)

# determine only the *output* layer names that we need from YOLO
ln = net.getLayerNames()
ln = [ln[i[0] - 1] for i in net.getUnconnectedOutLayers()]

"""
# Azure parameters
"""
# NOTE: you need a subscription key from Azure, place it in keys/azure.py with the identifier subscription_key
# Source: https://portal.azure.com
assert subscription_key
search_url = "https://api.cognitive.microsoft.com/bing/v7.0/images/search"

# Global Variables
if args["max"]:
    MAX_RESULTS = int(args["max"])
else:
    MAX_RESULTS = 250
if args["group"]:
    GROUP_SIZE = int(args["group"])
else:
    GROUP_SIZE = 50
if args["timeout"]:
    TIMEOUT = int(args["timeout"])
else:
    TIMEOUT = 30
if args["width"]:
    MIN_WIDTH = int(args["width"])
else:
    MIN_WIDTH = 0
if args["height"]:
    MIN_HEIGHT = int(args["height"])
else:
    MIN_HEIGHT = 0
if args["output"]:
    OUTPUT_DIR = args["output"]
else:
    OUTPUT_DIR = os.path.join('output', args["query"].replace(' ', '_').replace('\"', '').replace(':', '-').replace('.', 'p'))[:64]

"""
# Possible exceptions
"""
exception_set = [
    IOError, FileNotFoundError,
    requests.RequestException, requests.HTTPError,
    requests.ConnectionError, requests.Timeout,
    requests.exceptions.SSLError, requests.exceptions.ConnectTimeout
]
"""
# Possible image extensions
"""
extensions = [
    'jpg', 'jpeg', 'jpe', 'png', 'bmp',
    'pbm', 'pgm', 'ppm', 'sr', 'ras',
    'jp2', 'tiff', 'tif'
]
"""
# Create output directory
"""
# Source: https://stackoverflow.com/questions/273192/how-can-i-safely-create-a-nested-directory-in-python
if not os.path.exists(OUTPUT_DIR):
    os.makedirs(OUTPUT_DIR)

"""
# Search query
"""
term = args["query"]
headers = {"Ocp-Apim-Subscription-Key" : subscription_key}
params = {"q": args["query"], "imageType": "photo"}
logging.info("searching Bing API for '{}'".format(term))

try:
    response = requests.get(search_url, headers=headers, params=params)
    response.raise_for_status()
    search_results = response.json()

    estNumResults = min(search_results["totalEstimatedMatches"], MAX_RESULTS)
    logging.info("{} total results for '{}'".format(estNumResults, term))

    # Download images
    total = 0
    # Loop over total results in GROUP_SIZE batches
    for offset in range(0, estNumResults, GROUP_SIZE):
        logging.info("making request for group {}-{} of {}...".format(
            offset,
            offset+GROUP_SIZE,
            estNumResults
        ))
        params["offset"] = offset
        search = requests.get(search_url, headers=headers, params=params)
        search.raise_for_status()
        results = search.json()
        logging.info("saving images for group {}-{} of {}...".format(
            offset,
            offset+GROUP_SIZE,
            estNumResults
        ))
        # Loop over group batch
        for v in results["value"]:
            try:
                logging.info("fetching: {}".format(v["contentUrl"]))
                try:
                    r = requests.get(v["contentUrl"], timeout=TIMEOUT)
                except requests.ReadTimeout:
                    continue
                ext = v["contentUrl"][v["contentUrl"].rfind("."):]
                ext = ext.lower()

                if v["width"] < MIN_WIDTH or v["height"] < MIN_HEIGHT:
                    logging.warning("{}x{} is smaller than {}x{}, skipping: {}".format(
                        v["width"], v["height"],
                        MIN_WIDTH, MIN_HEIGHT,
                        v["contentUrl"])
                    )
                    continue
                # Check for valid extensions
                elif ext[1:] not in extensions:

                    ext = ext[:4]
                    if ext[1:] not in extensions:
                        # default to .jpg
                        ext = '.jpg'
                        logging.warning("invalid extension, assigning extension as jpg: {}".format(v["contentUrl"]))
                    else:
                        logging.warning("invalid extension, truncating extension: {}".format(v["contentUrl"]))
                # path in format 'outputPath/number.ext'
                p = os.path.sep.join(
                    [OUTPUT_DIR,
                     "{}{}".format(str(total).zfill(8), ext)]
                )
                # Write to disk
                f = open(p, "wb")
                f.write(r.content)
                f.close()
            except Exception as e:
                if type(e) in exception_set:
                    logging.warning("encountered exception: {}, skipping: {}".format(e, v["contentUrl"]))
                    continue
                else:
                    raise e
            """
            # verify it's a good image
            """
            image = cv2.imread(p)
            if image is None:
                logging.warning("deleting corrupted image: {}".format(p))
                os.remove(p)
                continue
            (H, W) = image.shape[:2]
            # construct a blob from the input frame and then perform a forward
            # pass of the YOLO object detector, giving us our bounding boxes
            # and associated probabilities
            blob = cv2.dnn.blobFromImage(
                image,
                1 / 255.0,  # sigma
                (416, 416),  # Resize to X by Y
                swapRB=True,  # BGR to RGB
                crop=False  # cropping
            )
            net.setInput(blob)
            layerOutputs = net.forward(ln)

            # initialize our lists of detected bounding boxes, confidences,
            # and class IDs, respectively
            boxes = []
            confidences = []
            classIDs = []

            # loop over each of the layer outputs
            for output in layerOutputs:
                # loop over each of the detections
                for detection in output:
                    # extract the class ID and confidence (i.e., probability)
                    # of the current object detection
                    scores = detection[5:]
                    classID = np.argmax(scores)
                    confidence = scores[classID]

                    # filter out weak predictions by ensuring the detected
                    # probability is greater than the minimum probability
                    if confidence > args["confidence"]:
                        # scale the bounding box coordinates back relative to
                        # the size of the image, keeping in mind that YOLO
                        # actually returns the center (x, y)-coordinates of
                        # the bounding box followed by the boxes' width and
                        # height
                        box = detection[0:4] * np.array([W, H, W, H])
                        (centerX, centerY, width, height) = box.astype("int")

                        # use the center (x, y)-coordinates to derive the top
                        # and and left corner of the bounding box
                        x = int(centerX - (width / 2))
                        y = int(centerY - (height / 2))

                        # update our list of bounding box coordinates,
                        # confidences, and class IDs
                        boxes.append([x, y, int(width), int(height)])
                        confidences.append(float(confidence))
                        classIDs.append(classID)

            # apply non-maxima suppression to suppress weak, overlapping
            # bounding boxes
            idxs = cv2.dnn.NMSBoxes(boxes, confidences, args["confidence"],
                                    args["threshold"])

            # Test to see if at least one person was detected
            if not any([LABELS[c] == 'person' for c in classIDs]):
                logging.warning("deleting image because no people found: {}".format(p))
                os.remove(p)
                continue

            # Found a good image, go to the next
            total += 1
except requests.HTTPError as e:
    # Sometimes you'll get this error if you entered the wrong subscription key
    # Source: https://blogs.msdn.microsoft.com/kwill/2017/05/17/http-401-access-denied-when-calling-azure-cognitive-services-apis/
    print(e)


class Solution:
    def isAnagram(self, s: str, t: str) -> bool:
        countS, countT = {},{}

        if len(s) != len(t):
            return False

        for i in range(len(s)):
            countS[s[i]] = 1 + countS.get(s[i],0)
            countT[t[i]] = 1 + countT.get(t[i],0)

        for c in countS:
            if countS.get(c) != countT.get(c):
                return False
        return True

s1 = Solution()
print(s1.isAnagram('rat','car'))


n = int(input())
res = 0
board=[0]*15
def recursive(x):
    global res
    if x == n:
        res += 1
        return
    for i in range(n):
        board[x] = i
        if check(x):
            recursive(x+1)

def check(x):
    for i in range(x):
        if board[x] == board[i] or abs(board[i]-board[x]) == x-i:
            return False
    return True

recursive(0)
print(res)


import numpy as np
import pandas as pd
import os
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import roc_auc_score
from sklearn.datasets import load_iris

os.chdir('/Users/alexanderchia/hc')

np.random.seed(0)

iris = load_iris()

df = pd.DataFrame(iris.data, columns = iris.feature_names)

df['species'] = pd.Categorical.from_codes(iris.target, iris.target_names)
df['target'] = iris.target

df['is_train'] = np.random.uniform(0, 1, len(df)) <= 0.75

train = df[df['is_train'] == True]
test = df[df['is_train'] == False]

features = df.columns[:4]

clf = RandomForestClassifier()

clf.fit(train[features], train['target'])

clf.predict(test[features])

probs = clf.predict_proba(test[features])
print(probs)

preds = clf.predict(test[features])

crosstab_ = pd.crosstab(test['target'], preds, rownames=['ACTUAL'], colnames=['PREDICTED'])
print(crosstab_)


import numpy as np
from yahoo_finance import Share
import copy
import transaction
import datetime
from dateutil import parser
import pprint

class Portfolio:
    """
    t = type Transaction

    Create new portfolio given old portfolio and a Transaction object. When
    last_portfolio is None, it means that our old portfolio was empty. Also,
    keep an instance variable start_date with the date of the transaction.
    """

    def __init__(self, t=None, last_portfolio=None, init_cash=None):
        if t is not None:
            # dictionary self.holdings: map from symbol (string) to qty (# shares)
            if last_portfolio is not None:
                self.holdings = copy.deepcopy(last_portfolio.holdings)
                if t.direction is not None:
                    qty = t.direction * t.quantity
                    if t.symbol in self.holdings:
                        self.holdings[t.symbol] += qty
                        if self.holdings[t.symbol] == 0:
                            del self.holdings[t.symbol]
                    else:
                        self.holdings[t.symbol] = qty
                        print("Shorting stock: {}".format(t.symbol) if t.direction == -1 else "Setting stock: {}".format(t.symbol))
                else:
                    self.holdings['cash'] += t.amt
            else: #this loop will never be activated with our current code
                print(t.direction, t.quantity, t.date)
                self.holdings = {}
                qty = t.direction * t.quantity
                self.holdings[t.symbol] = qty
                self.holdings['cash'] = init_cash
            if t.direction:
                comm = 0
                if t.comm:
                    comm = t.comm
                self.holdings['cash'] -= (qty * t.price + comm)
            self.start_date = t.date
        else:
            self.holdings = {}
            start = open('2015_start.csv', 'r')
            start_lines = start.readlines()
            for starts in start_lines:
                info = starts.split(',') #change to ints here
                if info[0] == 'cash':
                    self.holdings['cash'] = float(info[1])
                else:
                    self.holdings[info[0]] = int(info[1])
            # set the start date manually here
            self.start_date = parser.parse('12/31/2014')
            print(self.holdings)

    # storage for prices from yahoo finance
    # Prices is a dictionary of dictionaries; each inner dictionary consists of (date, value) pairs for a ticker
    prices = {}

    """
    Given date, calculate the value of the portfolio. Verifies that the
    date is after the start_date.
    """

    def calculateValue(self, date):
        value = 0.
        date = date.strftime('%Y-%m-%d')
        for sym,qty in self.holdings.items():
            if sym == 'cash':
                value += qty
            else:
                if sym not in Portfolio.prices:
                    stock = Share(sym)
                    # get prices until today
                    string_today = datetime.date.today().strftime('%Y-%m-%d')
                    # print(string_today, date, sym)
                    price_list = stock.get_historical(date, string_today)
                    # pprint.PrettyPrinter(depth = 6).pprint(price_list)
                    Portfolio.prices[sym] = {item['Date']: float(item['Close']) for item in price_list}

                # find price for the date of interest
                if date in Portfolio.prices[sym]:
                    close_price = Portfolio.prices[sym][date]
                    value += close_price * qty
                else:
                    # I still don't know why this would ever happen. What's going on?
                    print(date, sym)
                    return None
        return value


# Easy    Codewriting     300
#
# To prepare his students for an upcoming game, the sports coach decides to try some new training drills. To begin with, he lines them up and starts with the following warm-up exercise: when the coach says 'L', he instructs the students to turn to the left. Alternatively, when he says 'R', they should turn to the right. Finally, when the coach says 'A', the students should turn around.
#
# Unfortunately some students (not all of them, but at least one) can't tell left from right, meaning they always turn right when they hear 'L' and left when they hear 'R'. The coach wants to know how many times the students end up facing the same direction.
#
# Given the list of commands the coach has given, count the number of such commands after which the students will be facing the same direction.
#
# Example
#
# For commands = "LLARL", the output should be
# lineUp(commands) = 3.
#
# Let's say that there are 4 students, and the second one can't tell left from right. In this case, only after the second, third and fifth commands will the students face the same direction.
#
#
#
# Input/Output
#
# [execution time limit] 4 seconds (py3)
#
# [input] string commands
#
# String consisting of characters 'L', 'R' and 'A' only.
#
# Guaranteed constraints:
# 0 ≤ commands.length ≤ 35.
#
# [output] integer
#
# The number of commands after which students face the same direction.

def lineUp(commands):
    commands_list = list(commands)

    for i in range(len(commands_list)):
        if commands_list[i] == 'L' or commands_list[i] == 'R':
            del commands_list[i]
            commands_list.insert(i, False)

        else:
            del commands_list[i]
            commands_list.insert(i, True)

    result = 0
    commands_list.insert(0, True)

    for i in range(len(commands)):

        if commands_list[0] == True and commands_list[1] == True or commands_list[0] == False and commands_list[1] == False:
            del commands_list[1], commands_list[0]
            commands_list.insert(0, True)
            result += 1

        else:
            del commands_list[1], commands_list[0]
            commands_list.insert(0, False)

    return result


import csv
import numpy as np

#This program takes the original dataset and reformats it into:
#    hospitalData.csv:  A slightly cleaner file with wards and cites appened,
#                       and the time shown in weeks since start.
#    transfers.csv: A restructured format which all transfers are shown
#                   in the same row as the transfer source and target.
#    sourceTransfers.csv:   A file showing the number of transfers from each
#                           applicable ward per week.
#    targetTransfers.csv:  A file showing the number of transfers to each
#                          applicable ward per week.
#    transferProbability.csv:   A file showing the relative rates per transfer.

#This program must be run as a prerequisite to running patientFlowSim.py.
if __name__ == "__main__":
    with open('data/pone0185912s003.csv', 'rt') as f_input:
        csv_input = csv.reader(f_input, delimiter=',')
        header = next(csv_input)
        data = list(csv_input)

    myData = []
    for i in range(len(data)):
        currentData = []
        # The ward names and site codes are appended.
        currentData.append(data[i][0]+"."+data[i][1]) #Source
        currentData.append(data[i][2]+"."+data[i][3]) #Target
        currentData.append(int(data[i][4])) #Transfers
        currentYear = int(data[i][5]) #Year
        currentWeek = int(data[i][6]) #Week of Year
        # Convert weeks in a year to weeks from start.
        if currentYear == 2014:
            currentWeek = currentWeek-48
        if currentYear == 2015:
            currentWeek = currentWeek+3
        if currentYear == 2016:
            currentWeek = currentWeek + 52 + 3
        currentData.append(currentWeek)
        myData.append(currentData)

    #myData form is [Source, Target, Transfers, Week]
with open("data/hospitalData.csv",'w') as file:
    # Outputs data with combined names and weeks since start.
    writer = csv.writer(file, delimiter=',')
    writer.writerow(["Source","Target","Transfers","Week"])
    writer.writerows(myData)
Sources = []
Targets = []
for i in range(len(myData)):
    currentSource = myData[i][0]
    currentTarget = myData[i][1]
    if currentSource not in Sources:
        Sources.append(currentSource)
    if currentTarget not in Targets:
        Targets.append(currentTarget)

weeklyTransfers = [0]*82
for i in range(len(myData)):
    j = myData[i][3]
    weeklyTransfers[j]+=myData[i][2]

transferList = []
transferListLen = 0
transferListNo = []
for i in range(len(myData)):
    source = myData[i][0]
    target = myData[i][1]
    pairing = [source,target]
    week = myData[i][3]
    transferNo = myData[i][2]
    if pairing not in transferList:
        transferList.append(pairing)
        transferListLen += 1
        transferListNo.append([0]*82)
        transferListNo[transferListLen-1][week]+=transferNo
    else:
        transferListNo[transferList.index(pairing)][week]+=transferNo
with open("data/transfers.csv",'w') as file:
    #Shows data in form of Source, Target, Transfer in Week n
    writer = csv.writer(file, delimiter=',')
    writer.writerow(["Source","Target"]+list(range(82)))
    for i in range(len(transferList)):
        writer.writerow(transferList[i]+list(transferListNo[i]))

transferListNoNp = np.array(transferListNo)
sourceTransfers = np.array([np.array([0]*82)]*len(Sources))
for i in range(len(transferList)):
    source = transferList[i][0]
    j = Sources.index(source)
    sourceTransfers[j] += transferListNoNp[i]
with open("data/sourceTransfers.csv",'w') as file:
    # Shows transfers from each source per week
    writer = csv.writer(file, delimiter=',')
    writer.writerow(["Source"]+list(range(82)))
    for i in range(len(Sources)):
        writer.writerow([Sources[i]]+list(sourceTransfers[i]))

targetTransfers = np.array([np.array([0]*82)]*len(Targets))
for i in range(len(transferList)):
    target = transferList[i][1]
    j = Targets.index(target)
    targetTransfers[j] += transferListNoNp[i]
with open("data/targetTransfers.csv",'w') as file:
    # Shows transfers to each target per week.
    writer = csv.writer(file, delimiter=',')
    writer.writerow(["Target"]+list(range(82)))
    for i in range(len(Targets)):
        writer.writerow([Targets[i]]+list(targetTransfers[i]))

# Calculates the differents rates of transfer from each source-target.
probabilityMatrix = transferListNo
for i in range(len(transferList)):
    source = transferList[i][0]
    j = Sources.index(source)
    for k in range(82):
        if sourceTransfers[j][k]!=0:
            probabilityMatrix[i][k] = transferListNo[i][k]/sourceTransfers[j][k]

with open("data/transferProbability.csv",'w') as file:
    writer = csv.writer(file, delimiter=',')
    writer.writerow(["Source","Target"]+list(range(82)))
    for i in range(len(transferList)):
        writer.writerow(transferList[i]+list(probabilityMatrix[i]))


from ListNode import ListNode

class Solution(object):
    def partition(self, head, x):
        """
        :type head: ListNode
        :type x: int
        :rtype: ListNode
        """
        if not head or not head.next:
            return head
        dummy=ListNode(0)
        dummy.next=head
        low=dummy
        lownext=head
        highpre=dummy
        high=head
        while high:
            if high.val<x:
                if low.next==high:
                    low=low.next
                    lownext=low.next
                    highpre=high
                    high=high.next
                else:
                    low.next=high
                    temp=high.next
                    high.next=lownext
                    highpre.next=temp

                    high=temp
                    low=low.next
                    lownext=low.next
            else:
                highpre=high
                high=high.next
        return dummy.next

head=ListNode(1)
head.next=ListNode(4)
head.next.next=ListNode(3)
head.next.next.next=ListNode(2)
head.next.next.next.next=ListNode(5)
head.next.next.next.next.next=ListNode(2)

head2=Solution().partition(head,3)
while head2:
    print (head2.val)
    head2=head2.next


import urllib
import urlparse
import robotparser

class URLGatekeeper:
    """a class to track robots.txt rules across multiple servers"""
    def __init__(self):
        self.rpcache = {} # a dictionary of RobotFileParser objects, by domain
        self.robotsurl = ''
        __version__ = "1.371"
        self.urlopener = urllib.FancyURLopener()
        self.urlopener.version = "feedfinder/" + __version__ + " " + self.urlopener.version + " +http://www.aaronsw.com/2002/feedfinder/"
        self.urlopener.addheaders = [('User-agent', self.urlopener.version)]
        robotparser.URLopener.version = self.urlopener.version
        robotparser.URLopener.addheaders = self.urlopener.addheaders

    def _getrp(self, url):
        protocol, domain = urlparse.urlparse(url)[:2]
        if self.rpcache.has_key(domain):
            return self.rpcache[domain], self.robotsurl
        baseurl = '%s://%s' % (protocol, domain)
        self.robotsurl = urlparse.urljoin(baseurl, 'robots.txt')
        rp = robotparser.RobotFileParser(self.robotsurl)
        try:
            rp.read()
        except:
            pass
        self.rpcache[domain] = rp
        return rp, self.robotsurl

def get_crawl_access(url):
    urlgatekeeper = URLGatekeeper()
    try:
        rp, robotsurl = urlgatekeeper._getrp(url)
        crawl_status = rp.can_fetch('*', url)
        return crawl_status, robotsurl
    except:
        return True, ''


import numpy as np
import time
import matplotlib.pyplot as plt

def naive_calc(p,k):
    ws = np.array([],dtype=complex)
    for i in range(len(p)):
        w_i = np.exp(-2*np.pi*(i*1j)/k)
        ws = np.append(ws,w_i)
    ps = np.array([],dtype=complex)
    for i in range(len(p)):
        sumofp = complex(0.)
        for l in range(len(p)):
            w_i = ws[i] ** l
            sumofp = sumofp + w_i * p[l]
        ps = np.append(ps,sumofp)
    return ps

def fft_calc(p,ws):
    if (len(p) == 1):
        ps = np.array([complex(p[0] * ws[0])],dtype=complex)
    else :
        ps = np.empty(len(ws),dtype=complex)
        half_h = int(len(ws)/2)
        q = p[::2]
        s = p[1::2]
        qs = fft_calc(q,ws[::2])
        ss = fft_calc(s,ws[::2])
        for i in range(half_h):
            ps[i] = complex(qs[i] + ws[i] * ss[i])
        for i in range(half_h):
            ps[half_h+i] = complex(qs[i] - ws[i] * ss[i])
    return ps

def fft(p,k):
    ws = np.array([],dtype=complex)
    for i in range(len(p)):
        w_i = np.exp(-2*np.pi*(i*1j)/k)
        ws = np.append(ws,w_i)
    return fft_calc(p,ws)

def create_array(k):
    array = np.zeros(k,dtype=complex)
    for i in range(k):
        a = np.random.rand()
        b = np.random.rand()
        array[i] = a + b * 1j
    return array

if __name__ == "__main__" :
    fft_time = []
    naive_time = []
    for i in range(15):
        print("K = ", i)
        K = 2 ** i
        a = create_array(K)
        start = time.time()
        p1 = fft(a,K)
        elapsed_time = time.time() - start
        fft_time.append(elapsed_time)
        print ("fft_time:{0}".format(elapsed_time) + "[sec]")
        start = time.time()
        p2 = naive_calc(a,K)
        elapsed_time = time.time() - start
        naive_time.append(elapsed_time)
        print ("naive_time:{0}".format(elapsed_time) + "[sec]")
        print ("diff is ", np.linalg.norm(p1 - p2, ord = 2))
        print ("")

    plt.figure()
    plt.figure(figsize=(8,8))
    plt.plot(np.log2(fft_time),label="fft")
    plt.plot(np.log2(naive_time),label="naive")
    plt.axes().set_aspect('equal', 'datalim')
    plt.legend()
    plt.savefig("compare_time.png")


""" Module to take a water_level reading."""

# Raspi-sump, a sump pump monitoring system.
# Al Audet
# http://www.linuxnorth.org/raspi-sump/
#
# All configuration changes should be done in raspisump.conf
# MIT License -- http://www.linuxnorth.org/raspi-sump/license.html
import json
import os
import stat
import time

try:
    import ConfigParser as configparser  # Python2
except ImportError:
    import configparser  # Python3
from hcsr04sensor import sensor
from raspisump import log, alerts, heartbeat

config = configparser.RawConfigParser()
config.read("/home/pi/raspi-sump/raspisump.conf")

configs = {
    "critical_water_level": config.getint("pit", "critical_water_level"),
    "pit_depth": config.getint("pit", "pit_depth"),
    "temperature": config.getint("pit", "temperature"),
    "trig_pin": config.getint("gpio_pins", "trig_pin"),
    "echo_pin": config.getint("gpio_pins", "echo_pin"),
    "unit": config.get("pit", "unit"),
    "initialstate.enabled": config.getint("initialstate", "enabled"),
    "initialstate.bucket_name": config.get("initialstate", "bucket_name"),
    "initialstate.bucket_key": config.get("initialstate", "bucket_key"),
    "initialstate.access_key": config.get("initialstate", "access_key"),
    "initialstate.item_key": config.get("initialstate", "item_key"),
    "jsonfile.enabled": config.getint("jsonfile", "enabled"),
    "jsonfile.path": config.get("jsonfile", "path")
}

# If item in raspisump.conf add to configs dict. If not provide defaults.
try:
    configs["alert_when"] = config.get("pit", "alert_when")
except configparser.NoOptionError:
    configs["alert_when"] = "high"

try:
    configs["heartbeat"] = config.getint("email", "heartbeat")
except configparser.NoOptionError:
    configs["heartbeat"] = 0

def initiate_heartbeat():
    """Initiate the heartbeat email process if needed"""
    if configs["heartbeat"] == 1:
        heartbeat.determine_if_heartbeat()
    else:
        pass

def water_reading():
    """Initiate a water level reading."""
    pit_depth = configs["pit_depth"]
    trig_pin = configs["trig_pin"]
    echo_pin = configs["echo_pin"]
    temperature = configs["temperature"]
    unit = configs["unit"]

    value = sensor.Measurement(trig_pin, echo_pin, temperature, unit)

    try:
        raw_distance = value.raw_distance(sample_wait=0.3)
    except SystemError:
        log.log_errors(
            "**ERROR - Signal not received. Possible cable or sensor problem."
        )
        exit(0)

    return round(value.depth(raw_distance, pit_depth), 1)

def initialstate_stream_reading(reading):
    if "initialstate.enabled" not in configs or configs["initialstate.enabled"] != 1:
        return

    try:
        from ISStreamer.Streamer import Streamer
        streamer = Streamer(bucket_name=configs["initialstate.bucket_name"],
                            bucket_key=configs["initialstate.bucket_key"],
                            access_key=configs["initialstate.access_key"])
        streamer.log(key=configs["initialstate.item_key"], value=reading)
        streamer.flush()
        streamer.close()
        log.log_restarts("sent initialstate reading")
    except Exception as ex:
        log.log_errors("problem streaming reading to initialstate!")

def json_file_reading(reading):
    try:
        if "jsonfile.enabled" not in configs or configs["jsonfile.enabled"] != 1:
            return

        reading_for_json = {"level": reading,
                            "timestamp": time.time() * 1000}

        with open(configs["jsonfile.path"], mode='w') as jsonfile:
            json.dump(reading_for_json, jsonfile)
        os.chmod(configs["jsonfile.path"], stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
        log.log_restarts("wrote json file to {filename}".format(filename=configs["jsonfile.path"]))
    except Exception as ex:
        log.log_errors("problem streaming to json file!")

def water_depth():
    """Determine the depth of the water, log result and generate alert
    if needed.
    """

    critical_water_level = configs["critical_water_level"]
    water_depth = water_reading()

    if water_depth < 0.0:
        water_depth = 0.0
    log.log_reading(water_depth)
    initialstate_stream_reading(reading=water_depth)
    json_file_reading(reading=water_depth)

    if water_depth > critical_water_level and configs["alert_when"] == "high":
        alerts.determine_if_alert(water_depth)
    elif water_depth < critical_water_level and configs["alert_when"] == "low":
        alerts.determine_if_alert(water_depth)
    else:
        pass

    initiate_heartbeat()


def magfit(F,M):

    import numpy as np
    from scipy.optimize import curve_fit
    import matplotlib.pyplot as plt

    def func(x, a, b):
      return a*M + b

    y = func(M,2,4)

    popt, pcov = curve_fit(func,M,F)

    if(0):
        plt.figure()
        plt.plot(M,F,'bx',label='Data')
        plt.plot(M, func(M, *popt), 'k-',lw=1.5, label="fit")
        plt.title('Magnification Calibration')
        plt.xlabel('Magnification')
        plt.ylabel('$\Delta F$')
        plt.legend()
        plt.show()

    return popt[0],popt[1]


__author__ = 'similarface'
import sys, os, multiprocessing
from socket_stream_redirect import *
# def server1():
#     mypid = os.getpid()
#     conn = initListenerSocket()
#     file = conn.makefile('r')
#     for i in range(3):
#         data = file.readline().rstrip()
#         print('server %s got [%s]' % (mypid, data))

def server1():
    mypid=os.getpid()
    conn=initListenerSocket()
    file=conn.makefile('r')
    for i in range(3):
        data=file.readline().rstrip()
        print('Server %s got [%s] '%(mypid,data))

def client1():
    mypid=os.getpid()
    redirectOut()
    for i in range(3):
        print('client %s: %s' % (mypid, i))
        sys.stdout.flush()

if __name__ == '__main__':
    server=eval('server'+sys.argv[1])

    client=eval('client'+sys.argv[1])

    multiprocessing.Process(target=server).start()

    client()
    #import time; time.sleep(5)


import pathlib
from copy import deepcopy
from io import IOBase
from typing import Union, List

import attr
import paralleldots
import requests
import trio
from aylienapiclient import textapi
from requests import HTTPError

from srec_nlp.common import Query, Score
from srec_nlp.common import (_parallel, _parallel_async, _ensure_utf)
from srec_nlp.exc import RateError

class NLP:
	@_parallel
	@_ensure_utf
	def sentiment(self, text: Union[str, List[str], IOBase, pathlib.Path]) -> Query:
		"""
		:param text: Union[IOBase, str, Path, bytes] a file, pathtofile, bytes or string with the text.
					Function guarantees the value will be converted to text
		:return: Query
		"""
		raise NotImplemented

	@_parallel_async
	async def sentiment_async(self, text: Union[str, List[str], IOBase, pathlib.Path]):
		"""
		:param text: Union[IOBase, str, Path, bytes] a file, pathtofile, bytes or string with the text.
					Function guarantees the value will be converted to text
		:return: Query
		"""
		return await trio.run_sync_in_worker_thread(self.sentiment, text)

	def copy(self):
		return deepcopy(self)

@attr.s
class ParallelDots(NLP):
	"""
	   with ParallelDots(api_key) as nlp:
			   nlp.sentiment(text)
			   nlp.sentiment(bytes)
			   nlp.sentiment(open(path/to/text.txt,'r'))
			   nlp.sentiment(pathlib.Path('path/to/text.txt'))

	   """

	api_key: str = attr.ib()

	@api_key.validator
	def setkey(self, _, value):
		paralleldots.set_api_key(value)

	def __enter__(self):
		return self

	@_parallel
	@_ensure_utf
	def sentiment(self, text: Union[str, List[str], IOBase, pathlib.Path]) -> Query:
		"""
		Produces a sentiment score for query text using ParallelDots API
		:param text: Union[IOBase, str, Path, bytes] a file, pathtofile, bytes or string with the text.
					Function guarantees the value will be converted to text
		:return: Query
		"""
		try:
			response = paralleldots.sentiment(text)
			return Query(text, Score.fromtriplet(**response["sentiment"]))
		except KeyError as kerr:
			assert response
			if response.get('code', 0) == 429:
				raise RateError()
			else:
				print(response)
				raise kerr
		except HTTPError as err:
			print(response)
			raise err

	def __exit__(self, *exc):
		pass

class TextProcessing(NLP):
	"""
	with TextProcessing() as nlp:
			nlp.sentiment(text)
			nlp.sentiment(bytes)
			nlp.sentiment(open(path/to/text.txt,'r'))
			nlp.sentiment(pathlib.Path('path/to/text.txt'))

		# TextProcessing is still usable at this point.
		# So reusing it is still valid

		nlp.sentiment(text)
		nlp.sentiment(bytes)
		nlp.sentiment(open(path/to/text.txt,'r'))
		nlp.sentiment(pathlib.Path('path/to/text.txt'))
	"""

	_url = "http://text-processing.com/api/sentiment/"

	def __enter__(self):
		return self

	@_parallel
	@_ensure_utf
	def sentiment(self, text: Union[str, List[str], IOBase, pathlib.Path]) -> Query:
		"""
		Produces a sentiment score for query text using TextProcessing API
		:param text: Union[IOBase, str, Path, bytes] a file, pathtofile, bytes or string with the text.
					Function guarantees the value will be converted to text
		:return: Query
		"""
		data = dict(text=text)
		result = requests.post(self._url, data=data).json()
		result = result["probability"]
		result["positive"] = result["pos"]
		result["negative"] = result["neg"]
		del result["pos"]
		del result["neg"]
		try:
			return Query(content=text, score=Score.fromtriplet(**result))
		except (HTTPError, KeyError) as _:
			raise HTTPError()

	def __exit__(self, *exc):
		pass

try:
	try:
		import fastText
	except:
		import fasttext as fastText

	class FastText(NLP):
		"""
		model: str/pathlib.Path  the path to the model.

		# Usage example:

		with FastText(path/to/model.bin) as nlp:
			nlp.sentiment(text)
			nlp.sentiment(bytes).. Trio based queries
			nlp.sentiment(open(path/to/text.txt,'r'))
			nlp.sentiment(pathlib.Path('path/to/text.txt'))

		# at this point, nlp.sentiment WILL RAISE AN ERROR because the model has been
		# dealloc'd, to avoid this, use the following:

		nlp = FastText(path/to/model.bin)
		nlp.sentiment(text)
		nlp.sentiment(bytes)
		nlp.sentiment(open(path/to/text.txt,'r'))
		nlp.sentiment(pathlib.Path('path/to/text.txt'))

		"""

		def __init__(self, path: Union[str, pathlib.Path]):
			path = str(path).strip()
			path = pathlib.Path(path)
			self.path = path
			self.model = FastText._load_model(self.path)

		def __enter__(self):
			return self

		def __exit__(self, *vargs, **kwargs):
			del self.model

		@_parallel
		@_ensure_utf
		def sentiment(self, text: str) -> Query:
			"""
			Produces a sentiment score for query text using FastText API
			Note that FastText needs to be installed for this to work.
			:param text: Union[IOBase, str, Path, bytes] a file, pathtofile, bytes or string with the text.
						Function guarantees the value will be converted to text
			:return: Query
			"""
			labels, scores = self.model.predict(text, 1)
			label = labels[0]
			score = scores[0]
			if label == "__label__1":
				score = 1 - score
			score = Score.fromsingle(score)
			return Query(text, score)

		@staticmethod
		def _load_model(path: Union[str, pathlib.Path]):
			p = pathlib.Path(path).expanduser()
			if not p.is_file():
				raise ValueError(f"{p} is not a file.")
			try:
				return fastText.load_model(str(p))
			except Exception as e:
				print(e)
				raise ValueError(f"{p} is not a valid FastText model")

		def copy(self):
			return FastText(self.path)

except ImportError:
	pass

class Aylien(NLP):
	def __init__(self, appid, appkey):
		"""
		:param appid: string Aylien application id
		:param appkey: string Aylien application key

		with Aylien(id, key) as nlp:
			nlp.sentiment(text)
			nlp.sentiment(bytes)
			nlp.sentiment(open(path/to/text.txt,'r'))
			nlp.sentiment(pathlib.Path('path/to/text.txt'))

		# Aylien is still usable at this point.
		# So reusing it is still valid

		nlp.sentiment(text)
		nlp.sentiment(bytes)
		nlp.sentiment(open(path/to/text.txt,'r'))
		nlp.sentiment(pathlib.Path('path/to/text.txt'))
		"""
		self.client = textapi.Client(applicationId=appid, applicationKey=appkey)

	def __enter__(self):
		return self

	@_parallel
	@_ensure_utf
	def sentiment(self, text: Union[str, List[str], IOBase, pathlib.Path]) -> Query:
		"""
		Produces a sentiment score for query text using Aylien API
		:param text: Union[IOBase, str, Path, bytes] a file, pathtofile, bytes or string with the text.
					Function guarantees the value will be converted to text
		:return: Query
		"""
		sentiment = self.client.Sentiment(dict(text=text))
		try:
			confidence = sentiment["polarity_confidence"]
			klass = sentiment["polarity"]
			remaining = 1 - confidence
			vals = dict(positive=remaining / 2, negative=remaining / 2, neutral=remaining / 2)
			vals[klass] = confidence
			return Query(content=text, score=Score.fromtriplet(**vals))
		except (HTTPError, KeyError) as _:
			raise HTTPError()

	def __exit__(self, *exc):
		pass


import pytest
from selenium import webdriver
from selenium.webdriver.chrome.options import Options

def pytest_addoption(parser):
    parser.addoption('--language', action='store', default='ru', help='Chose language')

@pytest.fixture(scope="function")
def browser(request):

	# считываем язык и браузер (если есть)
	choosen_language = request.config.getoption("language")
    # инициализируем браузер chrome с нужными опциями
	opts = Options()
	opts.add_experimental_option('prefs', {'intl.accept_languages': choosen_language})
	print('\nstart Chrome browser...')
	browser = webdriver.Chrome(options=opts)
	yield browser
	# закрытие браузера после работы
	browser.quit()

unit3. Тестовые фреймворки/6. PyTest — параметризация, конфигурирование, плагины/Рецензии/2/Language_store-master/conftest.py


# Variables.py
# In this lesson we will be learning about Variables.
#
# Variables in programming are objects that you can store information
# in. They can be called in the program and have information put in
# them or read from them. Think of them like a container. You can put
# something in the container. You can also open the container and see
# whats inside it. You can transfer the contents of the container to
# another container.
#
# Variables are one of the main building blocks of
# programming. Lets try to use them.

def main():
	# Lets make a variable that contains the message "I am a variable."
	# We can call the variable whatever we want for now.
	var = "I am a variable."

	# Now we can do things with this variable. Lets try print it.
	# To print the variable, we just simply type "print" and then put
	# the variable inside the print function.
	print(var)

	# Now lets put another message in the variable. Lets see what
	# happens.
	var = "I am a different message"
	print(var)

	# Notice how the variable contents changed? This means that all the
	# contents were replaced with the new message.

	# You can also store numbers in the variables. Not just messages.
	newvar = 10
	print(newvar)

	newvar = 10.5
	print(newvar)

	# You can also copy the contents of a variable to another variable.
	original = 5
	varcopy = original
	original = 3
	varcopy = original
	print(original)
	print(varcopy)

	# Variables naming must follow rules. The rules are:
	# - Do not start with a number: 01variable
	# - Make sure that variable names relate to what is storing. If you
	#   are storing temperature readings in the variable call it names
	#   like:
	#	o temperature
	#   o temp
	#   o tempReading
	#
	#   NOT
	#   x atuhaeu
	#   x eeeeeee

	return 0

if __name__ == "__main__":
	import sys
	main()
	sys.exit()


import time
from collections import defaultdict
from datetime import datetime
from itertools import groupby

import nltk
import numpy as np
from nltk import TweetTokenizer
from nltk.corpus import stopwords
from nltk.tokenize import sent_tokenize

from opframe.db import Session
from opframe.models import search
from opframe.models.search import paginate
from opframe.plugins import plugin
from utils import rescale, render_template

stopwords_en = set(stopwords.words('english'))

# noinspection SpellCheckingInspection
@plugin('DocumentRenderer', 'view_renderer', 'Documents')
def render_documents(form, **kwargs):
    session = Session()
    print(form.values(), kwargs)
    results = search.search(session, **form.values())
    results, page, nb_pages = paginate(results, nb_pages=True, **form.values())
    form.page.data = page
    view = render_template('./templates/search/results_documents.html', form=form, nb_pages=nb_pages, results=results,
                           **kwargs)
    session.close()
    return view

# noinspection SpellCheckingInspection
@plugin('StatisticsRenderer', 'view_renderer', 'Statistics')
def render_statistics(form, **kwargs):
    session = Session()
    results = search.search(session, **form.values())
    # Create the corpus from the results
    tknzr = TweetTokenizer()
    texts = []
    for r in results:
        tokens = []
        for sent in sent_tokenize(r.text.strip()):
            tokens += tknzr.tokenize(sent.strip())
        texts.append(tokens)
    feature_summary = dict()
    feature_map = defaultdict(lambda: [])
    for r in results:
        for f in r.features:
            feature_map[f].append(r.features[f])
    # normalized_feature_map = feature_map
    summary = []
    # nfm = quantile_normalize(feature_map)
    nfm = rescale(feature_map)
    # nfm = feature_map
    for fname, data in nfm.items():
        hist, bin_edges = np.histogram(data, 20)
        hist, bin_edges = hist.tolist(), bin_edges.tolist()
        bins = [dict(x0=x, x1=y, length=z) for x, y, z in zip(bin_edges[:-1], bin_edges[1:], hist)]
        feature_summary[fname] = dict(bins=bins)
        quartiles = np.percentile(data, [25, 50, 75])
        summary += [dict(
            label=fname,
            values=dict(
                Q1=quartiles[0],
                Q2=quartiles[1],
                Q3=quartiles[2],
                min=min(data),
                max=max(data),
                outliers=[]
            )
        )]
    del feature_map
    corpus = nltk.TextCollection(texts)
    corpus.collocations(100)
    collocations = []
    # noinspection PyProtectedMember
    for collocation in corpus._collocations:
        temp = defaultdict(lambda: [])
        for r in results:
            if ' '.join(collocation) in r.text:
                for f in r.features:
                    temp[f].append(r.features[f])
        for k, v in temp.items():
            temp[k] = np.average(v)
        collocations += [(collocation, dict(temp))]
    results = {
        'vocabulary': corpus.vocab().most_common(100),
        'collocations': collocations,
        'features': feature_summary,
        'summary': summary
    }
    view = render_template('./templates/search/results_statistics.html', form=form, results=results, **kwargs)
    session.close()
    return view

# noinspection SpellCheckingInspection
@plugin('WordCloud', 'view_renderer', 'Word Cloud')
def render_wordcloud(form, **kwargs):
    session = Session()
    results = search.search(session, **form.values())
    # Create the corpus from the results
    tknzr = TweetTokenizer()
    texts = []
    for r in results:
        tokens = []
        for sent in sent_tokenize(r.text.strip()):
            tokens += [w for w in tknzr.tokenize(sent.strip()) if w.lower() not in stopwords_en]
        texts.append(tokens)
    corpus = nltk.TextCollection(texts)
    corpus.collocations(100)
    # noinspection PyProtectedMember
    results = {
        'vocabulary': [list(i) for i in corpus.vocab().most_common(1000)],
        'collocations': corpus._collocations,
    }
    view = render_template('./templates/search/results_wordcloud.html', form=form, results=results, **kwargs)
    session.close()
    return view

# noinspection SpellCheckingInspection
@plugin('SpaceTimeRender', 'view_renderer', 'Space Time')
def render_spacetime(form, **kwargs):
    session = Session()
    results = search.search(session, **form.values())
    timeline = []
    for hour, items in groupby(results, lambda x: [x.time.year, x.time.month, x.time.day, x.time.hour, 0, 0]):
        hour_start = time.mktime(datetime(*hour).timetuple())
        hour_end = hour_start + 3600
        timeline.append(
            {'id': hour, 'x': [hour_start, hour_end], 'y': sum(1 for _ in items)}
        )
    location_features = {'type': 'FeatureCollection', 'features': []}
    idx = 0
    for location, items in groupby(results, lambda x: x.location.display_name if x.location else None):
        if location is None or location == '':
            continue
        items = [i for i in items]
        if items[0].location.lat and items[0].location.lon:
            point = [items[0].location.lon, items[0].location.lat]
            location_features['features'].append({
                "type": "Feature",
                "id": '{}'.format(idx),
                'properties': {
                    'name': location,
                    "density": len(items),
                },
                'geometry': {
                    'type': 'Point',
                    'coordinates': point,
                },
            })
        idx += 1
    view = render_template('./templates/search/results_spacetime.html', form=form, **kwargs, timeline=timeline,
                           location_features=location_features)
    session.close()
    return view


# -*- coding: utf-8 -*-
"""
Created on Fri Feb  5 10:36:46 2021

@author: USER
"""
from mcpi.minecraft import Minecraft
import random
mc=Minecraft.create("104.199.150.71")
r=0
x=-555
y=86
z=275
myid=mc.getPlayerEntityId("MigUelLee")
r=random.randrange(0,5)
if r>2:
    mc.postToChat("你必須要被隔離")
    while True:
        mc.player.setTilePos(x,y,z)
elif r<2 or r==2:
    mc.postToChat("你不用被隔離")
    while True:
        x=-555
        y=86
        z=275

        mc.player.setTilePos(x,y,z)


import logging

from flask import Flask, request, jsonify

from hiku.graph import Graph, Root, Field, Option
from hiku.types import Record, Integer, TypeRef, String, Boolean
from hiku.engine import Engine
from hiku.executors.sync import SyncExecutor
from hiku.endpoint.graphql import GraphQLEndpoint

log = logging.getLogger(__name__)

def value_func(fields):
    return ['Hello World!' for _ in fields]

def action_func(fields):
    results = []
    for field in fields:
        print('action performed!', field.options)
        results.append(True)
    return results

DATA_TYPES = {
    'Point': Record[{
        'x': Integer,
        'y': Integer,
    }],
    'Data': Record[{
        'point': TypeRef['Point'],
    }],
}

QUERY_GRAPH = Graph([
    Root([
        Field('value', String, value_func),
    ]),
], data_types=DATA_TYPES)

MUTATION_GRAPH = Graph(QUERY_GRAPH.nodes + [
    Root([
        Field('action', Boolean, action_func,
              options=[Option('data', TypeRef['Data'])]),
    ]),
], data_types=DATA_TYPES)

app = Flask(__name__)

graphql_endpoint = GraphQLEndpoint(
    Engine(SyncExecutor()), QUERY_GRAPH, MUTATION_GRAPH,
)

@app.route('/graphql', methods={'POST'})
def handle_graphql():
    data = request.get_json()
    result = graphql_endpoint.dispatch(data)
    return jsonify(result)

def main():
    logging.basicConfig()
    app.run(port=5000)

if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-
# Copyright (C) 2017, Simone Marsili
# All rights reserved.
# License: BSD 3 clause
# Author: Simone Marsili (simomarsili@gmail.com)
"""
scikit-MSA: sequence data from sequence alignments
=======================================================================

The scikit-MSA (skmsa) package provides a simple interface between
biological sequence data - organized in a multiple sequence alignment
(MSA) file - and the Numpy/Scipy/pandas/scikit-learn ecosystem.
See docstrings for documentation.

Basic usage examples:
>>> import skmsa
>>> msa = skmsa.read('my_alignment_file.fa') # load sequence into a SequenceData object
>>> data = msa.as_array()   # export sequence data as a 2D numpy array
>>> df = msa.as_dataframe() # export sequence data as a pandas dataframe

"""

from __future__ import (absolute_import, division,
                        print_function, unicode_literals)
from builtins import (  # pylint: disable=redefined-builtin, unused-import
    bytes, dict, int, list, object, range, str,
    ascii, chr, hex, input, next, oct, open,
    pow, round, super,
    filter, map, zip)
import time
import logging
from skmsa.sd import SequenceData
from skmsa.parser import Parser
from skmsa.record import (Record, Alignment)
from skmsa.utils import get_version, config_loggers
read = SequenceData.read
load = SequenceData.load

__version__ = get_version()
__copyright__ = 'Copyright (C) 2017 Simone Marsili'
__license__ = 'BSD 3 clause'
__author__ = 'Simone Marsili (simo.marsili@gmail.com)'
__all__ = ['SequenceData',
           'read',
           'load',
           'Parser']

config_loggers()
logger = logging.getLogger(__name__)
now = time.strftime('%Y-%m-%d %H:%M:%S')
header = ('scikit-MSA (%s) - %s' % (__version__, now))
logger.info(header)
logger.info('=' * len(header))
import random
random.seed(1234)


import heapq

class BuildingSide:
  index = -1
  start = True
  height = -1

  def __init__(self, index, start, height):
    self.index = index
    self.start = start
    self.height = height

  def __eq__(self, other):
    return (
      self.index == other.index and
      self.start == other.start and
      self.height == other.height
    )

  def __lt__(self, other):
    # A lower value of index gets priority
    if self.index != other.index:
      return self.index < other.index

    # If the index values are the same, starting entry gets priority.
    if self.start != other.start:
      return self.start

    # If two buildings are starting at the same index,
    # then BuildingIndex with higher height gets priority.
    if self.start:
      return self.height > other.height

    # If two buildings are ending at the same index,
    # then BuildingIndex with lower height gets priority.
    else:
      return self.height < other.height

def get_bin_index(q, value):
  if not q:
    return 0

  b = 0
  e = len(q) - 1
  while b <= e:
    m = (b + e) // 2
    v = q[m][0]
    if v < value:
      b = m + 1
    elif v > value:
      e = m - 1
    else:
      return m

  if b < len(q) and q[b][0] == value:
    return b

  if e < len(q) and q[e][0] == value:
    return e

  return m

def ins_bin_point(q, idx, item):
  if idx >= len(q):
    q.append(item)
  else:
    if item[0] > q[idx][0]:
      idx += 1

    q.insert(idx, item)

def findSkyline(buildings):
    arr = []
    # 1. Pre-Processing
    for bldg in buildings:
      arr.append(BuildingSide(bldg[0], True, bldg[2]))
      arr.append(BuildingSide(bldg[1], False, bldg[2]))

    heapq.heapify(arr)

    height_count_q = [[0, 1]]
    max_height = 0
    horizon = []
    while arr:
      bldg_side = heapq.heappop(arr)
      # starting index signifies start of a buiding, so check if highest.
      if bldg_side.start:
        curr_i = get_bin_index(height_count_q, bldg_side.height)
        curr_count = height_count_q[curr_i][1] if height_count_q and height_count_q[curr_i][0] == bldg_side.height else 0
        if curr_count:
          height_count_q[curr_i][1] = curr_count + 1
        else:
          ins_bin_point(height_count_q, curr_i, [bldg_side.height, 1])
        # if tallest buiding detected then update the skyline.
        if bldg_side.height > max_height:
          max_height = bldg_side.height
          horizon.append([bldg_side.index, bldg_side.height])
      # end index signifies end of a buiding, so remove it from the heightCountQ,
      # and update skyline if highest.
      else:
        curr_i = get_bin_index(height_count_q, bldg_side.height)
        if height_count_q and height_count_q[curr_i][0] == bldg_side.height:
          if height_count_q[curr_i][1] == 1:
            del height_count_q[curr_i]
          else:
            height_count_q[curr_i][1] -= 1

        # update skyline, if tallest buiding ends.
        if max_height == bldg_side.height and height_count_q:
          last_val = height_count_q[-1]
          if max_height != last_val[0]:
            horizon.append([bldg_side.index, last_val[0]])

          height_count_q[-1][1] = last_val[1]
          max_height = last_val[0]

    return horizon


class Valid():
    def __init__(self):
        self._main = None
        self._required = []
        self._optional = []
        self._children = []
        self._valid = []

    def get_main(self):
        return self._main

    def set_main(self, main=None):
        if main is None:
            raise Exception('Valid set_main failed, main is {}'.format(main))
        if main not in self._required:
            raise Exception('Valid set_main failed, {} must be requered'.format(main))
        self._main = main

    def add_required(self, fields=None):
        for f in fields:
            self._required.append(f)

    def add_optional(self, fields=None):
        for f in fields:
            self._optional.append(f)

    def to_dict(self):
        d = {}
        d['required'] = []
        for r in self._required:
            d['required'].append(r)
        d['optional'] = []
        for r in self._optional:
            d['optional'].append(r)
        d['valid'] = []
        for s in self._valid:
            d['valid'].append(s.to_dict())
        if self._main:
            if isinstance(self._main, Valid):
                d['main'] = self._main.to_dict()
            else:
                d['main'] = self._main

        return d

    def test(self, data=None):
        if data is None:
            return False
        if self._main:
            if isinstance(self._main, str):
                if data.get(self._main) is None:
                    return False
            else:
                if not self._main.test(data):
                    return False

        for r in self._required:
            if isinstance(r, Valid):
                if not r.test(data):
                    return False
            else:
                if data.get(r) is None:
                    return False

        for s in self._valid:
            if isinstance(s.get_main(), str):
                if data.get(s.get_main()):
                    if not s.test(data):
                        return False
            else:
                if s.get_main().test(data):
                    if not s.test(data):
                        return False

        return True

    def add_valid(self, fields=None, info=None):
        s = Valid()
        required = []
        optional = []

        for f in fields:
            m = False
            if f in self._required:
                m = True

            if not m:
                if f in self._optional:
                    m = True
            if not m:
                if f in self._valid:
                    m = True
            if not m:
                print(self.to_dict())
                raise Exception('Valid add_valid failed, {} is not in requred or optional or valid list'.format(f))

        for f in fields:
            if f and info.get(f):
                i = info.get(f)
                if i.get('required'):
                    required.append(f)
                else:
                    optional.append(f)

            else:
                optional.append(f)
        s.add_required(fields=required)
        s.add_optional(fields=optional)
        main = None
        for f in fields:
            if f and info.get(f):
                i = info.get(f)
                if i.get('main'):
                    main = f
        if main is None:
            raise Exception('Valid add_valid failed, main field is {}'.format(main))
        s.set_main(main=main)

        self._valid.append(s)
        return s

    @classmethod
    def from_dict(self, hash_obj=None):
        if hash_obj is None or not isinstance(hash_obj, dict):
            return None

        s = Valid()
        if hash_obj.get('required'):
            if not isinstance(hash_obj.get('required'), list):
                raise Exception('Valid from_dict failed, required field must be list')
            s.add_required(fields=[f for f in hash_obj.get('required')])

        if hash_obj.get('optional'):
            if not isinstance(hash_obj.get('optional'), list):
                raise Exception('Valid from_dict failed, optional field must be list')
            s.add_optional(fields=[f for f in hash_obj.get('optional')])

        group_name_map = {}
        if hash_obj.get('groups'):
            if not isinstance(hash_obj.get('groups'), list):
                raise Exception('Valid from_dict failed, groups field must be list')

            keys = ['required', 'main', 'fields', 'name', 'parent']
            for g in hash_obj.get('groups'):
                if not isinstance(g, dict):
                    raise Exception('Valid from_dict failed, g type:{}'.format(type(g)))
                #if g.get('main') is None:
                    #raise Exception('Valid from_dict failed, g:{}'.format(g))
                if not isinstance(g.get('fields'), list):
                    raise Exception('Valid from_dict failed, g:{}'.format(g))
                if not isinstance(g.get('required'), list):
                    raise Exception('Valid from_dict failed, g:{}'.format(g))
                if not g.get('fields'):
                    raise Exception('Valid from_dict failed, g:{}'.format(g))
                if not g.get('required'):
                    raise Exception('Valid from_dict failed, g:{}'.format(g))
                if not isinstance(g.get('main'), str) or not isinstance(g.get('name'), str):
                    raise Exception('Valid from_dict failed, g:{}'.format(g))
                if group_name_map.get(g.get('name')):
                    raise Exception('Valid from_dict failed, {} is exists '+
                                    'in group_name_map'.fromat(g.get('name')))

                if g.get('parent'):
                    if g.get('parent') not in group_name_map:
                        raise Exception('Valid from_dict failed, g:{}'.format(g))

                info = {}
                for r in g.get('required'):
                    if r == g.get('main'):
                        d = r
                        if group_name_map.get(r):
                            d = group_name_map.get(r)

                        info[d] = {
                            "required": True,
                            "main": True
                        }
                    else:
                        d = r
                        if group_name_map.get(r):
                            d = group_name_map.get(r)

                        info[d] = {
                            "required": True
                        }
                parent = g.get('parent')
                fields = []
                for f in g.get('fields'):
                    d = f
                    if group_name_map.get(f):
                        d = group_name_map.get(f)
                    fields.append(d)

                if g.get('parent'):
                    parent = group_name_map.get(f)

                if parent:
                    _s = parent.add_valid(fields=fields, info=info)
                    group_name_map[g.get('name')] = _s
                else:
                    _s = s.add_valid(fields=fields, info=info)
                    group_name_map[g.get('name')] = _s
        return s

def run():
    s = Valid()

    s.add_required(fields=["src", "dst", "protocol", "workorder"])
    s.add_optional(fields=["sport", "dport", "vs_name", "pool", "nodes"])
    info = {
        "dport": {
            "required": True,
            "main": True
        },
        "sport": {
            "required": True
        },
        "vs_name": {
            "optional": True
        }
    }
    _s = s.add_valid(fields=["sport", "dport", "vs_name", "pool", "nodes"], info=info)
    info = {
        "vs_name":{
            "required": True,
            "main": True
        },
        "pool":{
            "required": True
        }
    }

    _ss = _s.add_valid(fields=["vs_name", "pool", "nodes"], info=info)
    info = {
        "pool": {
            "required": True,
            "main": True
        }
    }

    _ss.add_valid(fields=["pool", "nodes"], info=info)

    data = {}
    data['workorder'] = "A1234"
    data['src'] = '192.168.1.1/32'
    data['dst'] = '202.96.209.5/32'
    data['protocol'] = 'tcp'
    data['dport'] = '1234'
    data['sport'] = '1234'
    data['vs_name'] = "test_VS_NAME_1189"
    data['pool'] = 'test_pool_name'

    hash_obj = {
        "required": ["src", "dst", "protocol", "workorder"],
        "optional": ["vs_name", "pool", "nodes", "sport", "dport", "uuid", "A1", "A2", "A3", "A4"],

        "groups": [
            {
                "required": ["dport"],
                "main": "dport",
                "name": "port_group",
                "fields": ["dport", "sport", "A1", "A2", "A3", "A4"],
            },
            {
                "required": ["vs_name"],
                "main": "vs_name",
                "name": "F5",
                "fields": ["vs_name", "pool", "nodes"]
            },
            {
                "required": ["F5", "uuid"],
                "main": "F5",
                "name": "dport_f5",
                "fields": ["F5", "port_group", "uuid"]
            },
            {
                "required": ["A1"],
                "main": "A1",
                "name": "A1_valid",
                "fields": ["A1", "A2", "A3", "A4"],
                "parent": "port_group"
            }
        ]
    }

    s = Valid.from_dict(hash_obj=hash_obj)
    print(s.test(data=data))


#!/usr/bin/env python3
# @file
# user interface functions
#

from . import main
from . import op

## @brief     Extract, Convert and Save Markdown Style Comments From a File
#
#    This is merely a wrapper to excerpt(), modify_path() and pandoc().
#
#
# @param		file_name	The file from which the lines are to be extracted.
# @param		comment_character	The comment character of the file's language.
# @param		magic_character	The magic character marking lines as excerpts.
# @param		allow_pep8	Remove a leading single comment character and blank.
# @param		output_path	Set a new file name or an output directory.
# @param		postfix	Set the output file postfix.
# @param		prefix	Set the output file prefix.
# @param		run_pandoc	Run pandoc on the markdown file created?
# @param		pandoc_formats	The pandoc output formats to be used.
# @param		compile_latex	Compile the LaTeX file? Only considered if "tex" is part
#        of pandoc_formats.
# @return
#        0 if output generation was successful, 1 otherwise.
#

def excerpts(file_name, comment_character="#", magic_character="%",
             allow_pep8=True, output_path="", prefix="", postfix="",
             run_pandoc=True, compile_latex=False, pandoc_formats="tex"):
    status = 1
    with open(file_name) as infile:
        lines = infile.readlines()
    lines = main.excerpt(lines=lines,
                         comment_character=comment_character,
                         magic_character=magic_character,
                         allow_pep8=allow_pep8)
    md_file_name = main.modify_path(file_name=file_name,
                                    output_path=output_path,
                                    postfix=postfix,
                                    prefix=prefix,
                                    extension="md")
    md_file = open(md_file_name, "w")
    md_file.writelines(lines)
    md_file.close()
    if run_pandoc:
        status = op.pandoc(file_name=md_file_name, compile_latex=compile_latex,
                           formats=pandoc_formats)
    status = 0
    return status