import re
import pickle
import numpy as np
import pandas as pd
from nltk.stem import WordNetLemmatizer
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics import confusion_matrix, classification_report
from sklearn.linear_model import LogisticRegression
from gensim.parsing.preprocessing import remove_stopwords

# Dataset = UserName,ScreenName,Location,TweetAt,OriginalTweet,Sentiment
dataset = pd.read_csv(r'C:\Users\W10USER\PycharmProjects\MoodestSentiment\Corona_NLP_train.csv',
                      sep=",", encoding="ISO-8859-1")

# Gereksiz kolonları kaldır
dataset = dataset[['OriginalTweet', 'Sentiment']]

# Datayı listeleştirme
tweet, label = list(dataset['OriginalTweet']), list(dataset['Sentiment'])

# Emojileri tanımla
emojis = {':)': 'smile', ':-)': 'smile', ';d': 'wink', ':-E': 'vampire', ':(': 'sad',
          ':-(': 'sad', ':-<': 'sad', ':P': 'raspberry', ':O': 'surprised',
          ':-@': 'shocked', ':@': 'shocked', ':-$': 'confused', ':\\': 'annoyed',
          ':#': 'mute', ':X': 'mute', ':^)': 'smile', ':-&': 'confused', '$_$': 'greedy',
          '@@': 'eyeroll', ':-!': 'confused', ':-D': 'smile', ':-0': 'yell', 'O.o': 'confused',
          '<(-_-)>': 'robot', 'd[-_-]b': 'dj', ":'-)": 'sadsmile', ';)': 'wink',
          ';-)': 'wink', 'O:-)': 'angel', 'O*-)': 'angel', '(:-D': 'gossip', '=^.^=': 'cat'}

# Data temizlemek için fonksiyon
def preprocess(textdata):
    processedText = []
    wordLemm = WordNetLemmatizer()

    # Defining regex patterns.
    urlPattern = r"((http://)[^ ]*|(https://)[^ ]*|( www\.)[^ ]*)"
    userPattern = '@[^\s]+'
    alphaPattern = "[^a-zA-Z0-9]"
    sequencePattern = r"(.)\1\1+"
    seqReplacePattern = r"\1\1"

    for tweet in textdata:
        # Stringe çevir
        tweet = str(tweet)
        # Lowers letters
        tweet = tweet.lower()
        # Replace all URls with ' '
        tweet = re.sub(urlPattern, ' ', tweet)
        # Replace all emojis
        for emoji in emojis.keys():
            tweet = tweet.replace(emoji, emojis[emoji])
        # Replace @USERNAME to ' '.
        tweet = re.sub(userPattern, ' ', tweet)
        # Replace all non alphabets
        tweet = re.sub(alphaPattern, " ", tweet)
        # Replace repeating letters by 2 letters
        tweet = re.sub(sequencePattern, seqReplacePattern, tweet)
        # Checking stopwords and Lemmatizing
        tweetwords = ''
        for word in tweet.split():
            if len(word) > 1:
                word = wordLemm.lemmatize(word)
                tweetwords += (word + ' ')

        tweet = remove_stopwords(tweetwords)

        processedText.append(tweet)

    return processedText

processedText = preprocess(tweet)

# Data bölümlendirmesi
X_train, X_test, y_train, y_test = train_test_split(processedText, label,
                                                    test_size=0.2, random_state=0)

# Data dönüşümü, vektöre çeviriyoruz
vectoriser = TfidfVectorizer(ngram_range=(1, 2), max_features=500000)
vectoriser.fit(X_train)

X_train = vectoriser.transform(X_train)
X_test = vectoriser.transform(X_test)

# Modelleme
def model_Evaluate(model):
    # Test dataseti için değerleri öngör
    y_pred = model.predict(X_test)

    # Karışıklık matrisini hesapla ve çiz
    cf_matrix = confusion_matrix(y_test, y_pred)

    categories = ['Extremely Negative', 'Negative', 'Neutral', 'Positive', 'Extremely Positive']
    group_names = ['True Neg', 'False Pos', 'False Neg', 'True Pos']
    group_percentages = ['{0:.2%}'.format(value) for value in cf_matrix.flatten() / np.sum(cf_matrix)]

    labels = [f'{v1}\n{v2}' for v1, v2 in zip(group_names, group_percentages)]
    labels = np.asarray(labels).reshape(2, 2)
    # array (varsayılan olarak) nesnenin bir kopyasını oluşturur, asarray gerekmedikçe oluşturmaz.

    sns.heatmap(cf_matrix, annot=True, cmap='Blues', fmt='',
                xticklabels=categories, yticklabels=categories)

    plt.xlabel("Predicted values", fontdict={'size': 14}, labelpad=10)
    plt.ylabel("Actual values", fontdict={'size': 14}, labelpad=10)
    plt.title("Confusion Matrix", fontdict={'size': 18}, pad=20)

LRmodel = LogisticRegression(C=5, max_iter=1000, n_jobs=-1)
LRmodel.fit(X_train, y_train)

model_Evaluate(LRmodel)

file = open('vectoriser-ngram-(1,2).pickle', 'wb')
pickle.dump(vectoriser, file)
file.close()

def load_models():
    # Load the vectoriser.
    file = open(r'C:\Users\W10USER\PycharmProjects\Moodest2\vectoriser-ngram-(1,2).pickle', 'rb')
    vectoriser = pickle.load(file)
    file.close()
    # Load the LR Model.
    file = open(r'C:\Users\W10USER\PycharmProjects\Moodest2\Sentiment-LR.pickle', 'rb')
    LRmodel = pickle.load(file)
    file.close()

    return vectoriser, LRmodel

# Öngörü fonksiyonu
def predict(vectoriser, model, tweet):
    # Predict the sentiment
    textdata = vectoriser.transform(preprocess(tweet))
    label = model.predict(textdata)

    # Duygu ve Tweetlerden bir liste oluştur
    data = []
    for tweet, pred in zip(tweet, label):
        data.append((tweet, pred))

    # Listeyi pandas dataframeine dönüştür
    df = pd.DataFrame(data, columns=['tweet', 'label'])
    df = df.replace([-1, 0, 1, 2, 3], ['Extremely Negative', 'Negative', 'Neutral', 'Positive', 'Extremely Positive'])
    return df

if __name__ == "__main__":
    # Çıkana kadar programda kalmanı sağlayan while döngüsü
    key = 0
    while key == 0:
        platform = input('''       Select a platform
                1) TWITTER
                2) REDDIT
                3) Sentence

                Press 9 for quit.''')

        if platform == '9':
            print("See you later...")
            break

        if platform == '1':
            dataset = pd.read_csv(
                r'C:\Users\W10USER\PycharmProjects\MoodestSentiment\Workflow 154325 - twitter-698589.csv',
                sep=',')

            dataset = dataset[['text']]
            text = list(dataset['text'])

            word = input("Enter a Hashtag or word")
            preprocess(text)
            processedText = preprocess(text)
            processedText = [s for s in processedText if word in s]

            df = predict(vectoriser, LRmodel, processedText)

            plt.figure(figsize=(10, 5))
            sns.countplot(x='label', data=df,
                          order=['Extremely Negative', 'Negative', 'Neutral', 'Positive', 'Extremely Positive'])
            plt.title("Sentiment")
            plt.ylabel("Count", fontsize=12)
            plt.xlabel("Sentiments", fontsize=12)
            plt.show()

        elif platform == '2':

            input('''       We are currently only able to show data related to Covid-19.
                        Please press enter to continue.''')
            print("Loading...")
            dataset = pd.read_csv(r'C:\Users\W10USER\PycharmProjects\MoodestSentiment\coronavirus_reddit_posts.csv',
                                  sep=',')

            dataset = dataset[['body']]
            text = list(dataset['body'])

            preprocess(text)
            processedText = preprocess(text)

            df = predict(vectoriser, LRmodel, processedText)

            plt.figure(figsize=(10, 5))
            sns.countplot(x='label', data=df,
                          order=['Extremely Negative', 'Negative', 'Neutral', 'Positive', 'Extremely Positive'])
            plt.title("Sentiment")
            plt.ylabel("Count", fontsize=12)
            plt.xlabel("Sentiments", fontsize=12)
            plt.show()

        elif platform == '3':
            sentence = []
            text = input("Please enter a sentence:")
            sentence.append(text)

            df = predict(vectoriser, LRmodel, sentence)
            print(df.head())


from time import time
import math
import numpy as np
import pandas as pd
from pandas import DataFrame
from sklearn import preprocessing
from sklearn.linear_model import LogisticRegression
from sklearn.cross_validation import train_test_split
from numpy import loadtxt, where
from pylab import scatter, show, legend, xlabel, ylabel
start1=time()
min_max_scaler = preprocessing.MinMaxScaler(feature_range=(-1,1))
df = pd.read_csv("dataset.csv", header=0)

df.columns = ["age","sex","cp","trestbps","chol","fbs","restecg","thalach","exang","oldpeak","slope","ca","thal","num"]

x = df["num"].map(lambda x: float(x))

X = df[["age","sex","cp","trestbps","chol","fbs","restecg","thalach","exang","oldpeak","slope","thal"]]
X = np.array(X)
X = min_max_scaler.fit_transform(X)
Y = df["num"].map(lambda x: float(x))
Y = np.array(Y)

X_train,X_test,Y_train,Y_test = train_test_split(X,Y,test_size=0.33)

clf = LogisticRegression()
clf.fit(X_train,Y_train)
print ("score Scikit learn: ", clf.score(X_test,Y_test))
stop1=time()
pos = where(Y == 1)
neg = where(Y == 0)
scatter(X[neg, 0], X[neg, 1], marker='x', c='r')
scatter(X[pos, 0], X[pos, 1], marker='o', c='g')
xlabel('Features')
ylabel('Disease')
legend(['No', 'Yes'])
show()

def Sigmoid(z):
	G_of_Z = float(1.0 / float((1.0 + math.exp(-1.0*z))))
	return G_of_Z

def Hypothesis(theta, x):
	z = 0
	for i in range(len(theta)):
		z += x[i]*theta[i]
	return Sigmoid(z)

def Cost_Function(X,Y,theta,m):
	sumOfErrors = 0
	for i in range(m):
		xi = X[i]
		hi = Hypothesis(theta,xi)
		error = 0
		if Y[i] == 1:
			error = Y[i] * math.log(hi)
		elif Y[i] == 0:
			error = (1-Y[i]) * math.log(1-hi)
		sumOfErrors += error
	const = -1/m
	J = const * sumOfErrors
	return J

def Cost_Function_Derivative(X,Y,theta,j,m,alpha):
	sumErrors = 0
	for i in range(m):
		xi = X[i]
		xij = xi[j]
		hi = Hypothesis(theta,X[i])
		error = (hi - Y[i])*xij
		sumErrors += error
	m = len(Y)
	constant = float(alpha)/float(m)
	J = constant * sumErrors
	return J

def Gradient_Descent(X,Y,theta,m,alpha):
	new_theta = []
	constant = alpha/m
	for j in range(len(theta)):
		CFDerivative = Cost_Function_Derivative(X,Y,theta,j,m,alpha)
		new_theta_value = theta[j] - CFDerivative
		new_theta.append(new_theta_value)
	return new_theta

def Logistic_Regression(X,Y,alpha,theta,num_iters):
	m = len(Y)
	for x in range(num_iters):
		new_theta = Gradient_Descent(X,Y,theta,m,alpha)
		theta = new_theta
		if x % 100 == 0:
			#here the cost function is used to present the final hypothesis of the model in the same form for each gradient-step iteration
			Cost_Function(X,Y,theta,m)
			#print ("\n\ntheta ", theta)
			#print ("cost is ", Cost_Function(X,Y,theta,m))
	Declare_Winner(theta)

def Declare_Winner(theta):
	score = 0
	winner = ""

	scikit_score = clf.score(X_test,Y_test)
	length = len(X_test)
	for i in range(length):
		prediction = round(Hypothesis(X_test[i],theta))
		answer = Y_test[i]
		if prediction == answer:
			score += 1

	my_score = float(score) / float(length)
	print ("\nMy Accuracy: ", my_score*100)
	print ("Scikits Accuracy: ", scikit_score*100)

initial_theta = [0,0,0,0,0,0,0,0,0,0,0,0]
alpha = 0.00001
iterations = 1000
start2=time()
Logistic_Regression(X,Y,alpha,initial_theta,iterations)
stop2=time()
print("Scikit learn time => ",(stop1-start1)*1000,"Microsecond")
print("My time => ",(stop2-start2)*1000,"Microsecond")


from keras.layers import Dense, Flatten, Conv2D, Permute, Conv2DTranspose, Input, concatenate, Reshape, Cropping1D
from keras.models import Sequential, Model
from keras.models import model_from_json
from keras.optimizers import Adam
from rl.agents.dqn import DQNAgent
from rl.memory import SequentialMemory
from rl.policy import LinearAnnealedPolicy, EpsGreedyQPolicy

from atari_processor import AtariProcessor
from constants import *

def load_model(filename='models/model_raw_images.'):
    model_json = open(filename + 'json', 'r')
    loaded_model_json = model_json.read()
    model_json.close()
    loaded_model = model_from_json(loaded_model_json)
    loaded_model.load_weights(filename + 'h5')
    print("Loaded model from disk")
    return loaded_model

def get_cnn_model(input_shape=INPUT_SHAPE_CNN, nb_actions=2):
    model = Sequential()
    model.add(Permute((2, 3, 1), input_shape=input_shape))
    model.add(Conv2D(32, (8, 8), strides=(4, 4), activation='relu'))
    model.add(Conv2D(64, (4, 4), strides=(2, 2), activation='relu'))
    model.add(Conv2D(64, (3, 3), strides=(1, 1), activation='relu'))
    model.add(Flatten())
    model.add(Dense(512, activation='relu'))
    model.add(Dense(nb_actions, activation='linear'))
    return model

def get_agent_from_model(model, nb_actions, input_shape):
    memory = SequentialMemory(limit=50000, window_length=WINDOW_LENGTH)
    processor = AtariProcessor(input_shape=input_shape, preprocess_images=PREPROCESS)
    policy = LinearAnnealedPolicy(EpsGreedyQPolicy(), attr='eps', value_max=EPS_MAX, value_min=EPS_MIN,
                                  value_test=EPS_TEST, nb_steps=1000000)
    dqn = DQNAgent(model=model, nb_actions=nb_actions, policy=policy, memory=memory, processor=processor,
                   batch_size=BATCH_SIZE, nb_steps_warmup=50000, gamma=GAMMA, target_model_update=10000,
                   train_interval=4, delta_clip=1.)
    dqn.compile(Adam(lr=LEARNING_RATE), metrics=['mae'])
    return dqn

def adapt_model_to_alife(model, input_shape=(10, 1), vision_shape=(1, 3, 3), energy_shape=(1, 1, 1)):
    bug_input = Input(shape=input_shape)

    model_vision = Sequential()
    model_vision.add(Cropping1D(cropping=(0, 1), input_shape=input_shape))
    model_vision.add(Reshape(target_shape=vision_shape))
    model_vision.add(Conv2DTranspose(4, (SHRUNKEN_SHAPE[0], int(SHRUNKEN_SHAPE[1] / 2) - 1),
                                     input_shape=vision_shape, padding='valid'))
    model_vision.add(Permute((3, 1, 2)))
    vision_out = model_vision(bug_input)

    model_energy = Sequential()
    model_energy.add(Cropping1D(cropping=(9, 0), input_shape=input_shape))
    model_energy.add(Reshape(target_shape=energy_shape))
    model_energy.add(Conv2DTranspose(4, (SHRUNKEN_SHAPE[0], int(SHRUNKEN_SHAPE[1] / 2) - 1),
                                     input_shape=energy_shape, padding='valid'))
    model_energy.add(Permute((3, 1, 2)))
    energy_out = model_energy(bug_input)

    merged_tensor = concatenate([vision_out, energy_out], axis=-1)

    model.pop()
    model.add(Reshape((1, 512, 1)))
    model.add(Conv2D(SPEED_ACTION_POSSIBILITIES, (1, 512 - ANGLE_ACTION_POSSIBILITIES + 1), name='conv_2d_alife'))

    output = model(merged_tensor)

    complete_model = Model(inputs=bug_input, outputs=output)

    return complete_model


import pandas as pd

def add_num_games(df):
    num_games = pd.read_csv("data/useful/num_matches.csv")
    df = pd.merge(df, num_games, on="playerid", how="inner")
    return df

df = pd.read_csv("data/useful/merged_data.csv")

leagues_df = df.drop(columns=["result"])
leagues_df = df.groupby('playerid')['league'].agg(lambda x:x.value_counts().idxmax()).reset_index()
leagues_df = pd.get_dummies(data=leagues_df, columns=["league"])
print(leagues_df)

df = df[['playerid'] + df.select_dtypes(include='number').columns.tolist()]
df = df.drop(columns=["result"])
df = df.groupby('playerid').mean().reset_index()
df = add_num_games(df)

seq_elo = pd.read_csv("data/useful/sequential_elos.csv")
seq_elo.rename(columns={"elo": "seq_elo"}, inplace=True)

df = pd.merge(df, seq_elo, on="playerid", how="inner")

elo = pd.read_csv("data/useful/elos.csv")

df = pd.merge(df, elo, on="playerid", how="inner")

df = pd.merge(df, leagues_df, on="playerid", how="inner")

print(df)

df.to_csv("data/useful/player_data.csv", index=False)


# Modern Art
import math
from copy import copy

def possible_arrangements(artists):
    total_paintings = 0
    duplicates_per_arrangement = 1
    # Find all arrangements, then remove duplicates
    for artist in artists:
        total_paintings += artist
        duplicates_per_arrangement *= math.factorial(artist) # Different orders of indistinguishable = same
    return math.factorial(total_paintings) // duplicates_per_arrangement

def alphabet(arrangement):
    result = ""
    for index in arrangement:
        result += chr(65 + index) # From A
    return result

inputted = list(map(int, input().split()))
artists = inputted[:-1]
find_nth = inputted[-1]
print(possible_arrangements(artists), "arrangements")
length = sum(artists)

arrangement = []

for spaces_left in range(length-1, -1, -1): # Down to 0
    # Choose next digit - batch remove subtrees
    can_batch_remove = 0
    digit_to_use = -1
    while(can_batch_remove < find_nth):
        # Batch remove
        find_nth -= can_batch_remove
        digit_to_use += 1
        # Find size of next batch remove
        temp_artists = copy(artists)
        temp_artists[digit_to_use] -= 1
        if(temp_artists[digit_to_use] >= 0):
            can_batch_remove = possible_arrangements(temp_artists)
        else:
            can_batch_remove = 0 # None left from artist
        # print(digit_to_use, "temp", temp_artists, can_batch_remove, find_nth)

    artists = temp_artists
    # print(artists)

    arrangement.append(digit_to_use)

print("Arrangement: ", alphabet(arrangement))


import gc

import os

import numpy as np

import pandas as pd

import lightgbm as lgb

from sklearn.metrics import roc_auc_score
train_features_path = '../input/baseline-features/train_features.csv'

test_features_path = '../input/baseline-features/test_features.csv'

train = pd.read_csv(train_features_path)

test = pd.read_csv(test_features_path)
train = train.sort_values('TransactionDT')

test = test.sort_values('TransactionDT')
import matplotlib.pyplot as plt

fig, axs = plt.subplots(1,2, figsize=(16,4))

train.groupby(['TransactionDT'])['TransactionDT'].size().plot(ax=axs[0])

test.groupby(['TransactionDT'])['TransactionDT'].size().plot(ax=axs[1])
del test

gc.collect()
split_perc = [p*0.01 for p in range(100)]

y_means_train, y_means_valid = [],[]

for p in split_perc:

    idx = int(p*len(train))

    y_means_train.append(train['isFraud'][:idx].mean())

    y_means_valid.append(train['isFraud'][idx:].mean())
fig, ax = plt.subplots(figsize=(16,4))

ax.plot(split_perc, y_means_train, label='train')

ax.plot(split_perc, y_means_valid, label='valid')
split_perc_df = pd.DataFrame({'perc':split_perc,'train':y_means_train, 'valid':y_means_valid})

split_perc_df['diff'] = abs(split_perc_df['train']-split_perc_df['valid'])

split_perc_df.sort_values('diff').head()
0.13*len(train), 0.26*len(train)
def sample_negative_class(train, perc):

    train_pos = train[train.isFraud==1]

    train_neg = train[train.isFraud==0].sample(frac=perc)

    train = pd.concat([train_pos, train_neg], axis=0)

    train = train.sort_values('TransactionDT')

    return train
def fit_predict(train, valid, model_params, training_params):

    X_train = train.drop(['isFraud', 'TransactionDT', 'TransactionID'], axis=1)

    y_train = train['isFraud']

    X_valid = valid.drop(['isFraud', 'TransactionDT', 'TransactionID'], axis=1)

    y_valid = valid['isFraud']

    trn_data = lgb.Dataset(X_train, y_train)

    val_data = lgb.Dataset(X_valid, y_valid)

    clf = lgb.train(model_params, trn_data,

                    training_params['num_boosting_rounds'],

                    valid_sets = [trn_data, val_data],

                    early_stopping_rounds = training_params['early_stopping_rounds'],

                    verbose_eval=False

                   )

    train_preds = clf.predict(X_train, num_iteration=clf.best_iteration)

    valid_preds = clf.predict(X_valid, num_iteration=clf.best_iteration)

    return train_preds, valid_preds
idx_split = int(0.74*len(train))

train_split, valid_split = train[:idx_split], train[idx_split:]
model_params = {'num_leaves': 256,

                  'min_child_samples': 79,

                  'objective': 'binary',

                  'max_depth': 15,

                  'learning_rate': 0.05,

                  "boosting_type": "gbdt",

                  "subsample_freq": 3,

                  "subsample": 0.9,

                  "bagging_seed": 11,

                  "metric": 'auc',

                  "verbosity": -1,

                  'reg_alpha': 0.3,

                  'reg_lambda': 0.3,

                  'colsample_bytree': 0.9

                 }

training_params = {'num_boosting_rounds':1000,

                   'early_stopping_rounds':100,

               }
train_sample_perc = [p*0.05 for p in range(1,20,1)]

train_scores, valid_scores = [],[]

for perc in train_sample_perc:

    print('processing for perc {}'.format(perc))

    train_sample = sample_negative_class(train_split, perc)

    train_preds, valid_preds = fit_predict(train_sample, valid_split, model_params, training_params)

    train_score = roc_auc_score(train_sample['isFraud'], train_preds)

    valid_score = roc_auc_score(valid_split['isFraud'], valid_preds)

    print(perc, train_score, valid_score)

    train_scores.append(train_score)

    valid_scores.append(valid_score)
fig, axs = plt.subplots(2,1, figsize=(16,4))

axs[0].plot(train_sample_perc, train_scores, label='train')

axs[1].plot(train_sample_perc, valid_scores, label='valid')
sample_perc_df = pd.DataFrame({'perc':train_sample_perc,'train':train_scores, 'valid':valid_scores})

sample_perc_df.sort_values('valid', ascending=False).head()


import pandas as pd
import numpy as np

import torch
import joblib

from sklearn import preprocessing
from sklearn import model_selection

from transformers import AdamW, BertForTokenClassification
from transformers import get_linear_schedule_with_warmup

import config
import dataset
import engine
from model import EntityModel

def process_data(data_path):
    df = pd.read_csv(data_path)

    enc_tag = preprocessing.LabelEncoder()

    df.tag = enc_tag.fit_transform(df['tag'])

    titles = df.groupby('title#')['word'].apply(list).values
    tags = df.groupby('title#')['tag'].apply(list).values
    return titles, tags, enc_tag

if __name__ == '__main__':
    titles, tags, enc_tag = process_data(config.TRAINING_FILE)

    meta_data = {
        'enc_tag': enc_tag
    }
    joblib.dump(meta_data,'meta.bin')
    num_tag = len(list(enc_tag.classes_))

    (
        train_titles,
        valid_titles,
        train_tag,
        valid_tag
    ) = model_selection.train_test_split(titles, tags, random_state=42,test_size=0.1)

    train_dataset = dataset.EntityDataset(texts = train_titles, tags=train_tag)
    print(train_dataset[0])
    print(train_dataset[1])
    print(train_dataset[9])
    train_data_loader = torch.utils.data.DataLoader(
        train_dataset, batch_size=config.TRAIN_BATCH_SIZE, num_workers=4
    )

    valid_dataset = dataset.EntityDataset(texts = valid_titles, tags=valid_tag)
    print(valid_dataset[3])
    print(valid_dataset[8])
    valid_data_loader = torch.utils.data.DataLoader(
        valid_dataset, batch_size=config.VALID_BATCH_SIZE, num_workers=1
    )

    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    model = EntityModel(num_tag=num_tag)

    model.to(device)

    param_optimizer = list(model.named_parameters())
    no_decay = ['bias', 'LayerNorm.bias','LayerNorm.weight']
    optimizer_parameters=[
        {
            'params':[
                p for n,p in param_optimizer if not any( nd in n for nd in no_decay)
            ],
            'weight_decay':0.001,
        },
        {
            'params':[
                p for n,p in param_optimizer if any(nd in n for nd in no_decay)
            ],'weight_decay':0.0,
        }
    ]

    num_train_steps = int(len(train_titles)/config.TRAIN_BATCH_SIZE * config.EPOCHS)
    optimizer = AdamW(optimizer_parameters, lr=3e-5)
    scheduler = get_linear_schedule_with_warmup(
        optimizer, num_warmup_steps=0, num_training_steps=num_train_steps
    )

    best_loss = np.inf
    for epoch in range(config.EPOCHS):
        train_loss = engine.train_fn(train_data_loader, model, optimizer, device, scheduler)
        valid_loss = engine.eval_fn(valid_data_loader,model,device)
        print(f'Train loss {train_loss} Valid Loss ={valid_loss}')
        if valid_loss < best_loss:
            torch.save(model.state_dict(), config.MODEL_PATH)
            best_loss = valid_loss


import json

import SimpleITK as sitk
import cv2 as cv
import numpy as np
from scipy import ndimage
from skimage import measure

from pipeline.constants import DirectoryStructure as ds, FileNames as fn
from pipeline.logger import log

def auto_canny(image: np.ndarray, sigma: float = 0.01) -> np.ndarray:
    v = np.median(image)

    return cv.Canny(image, int(max(0, (1.0 - sigma) * v)), int(min(255, (1.0 + sigma) * v)))

def anomaly_detection() -> None:
    crop_val = 20
    volume = sitk.ReadImage(ds.analysis.path(fn.body_mask.value))
    x, y, z = volume.GetSize()
    volume = sitk.GetArrayFromImage(volume)

    c = np.zeros([1, y])
    for d in range(y):
        c[0, d] = volume[:, d, :].sum()
    coronal = volume[::-1, c.argmax(), :]

    s = np.zeros([1, x])
    for d in range(x):
        s[0, d] = volume[:, :, d].sum()
    sagittal = volume[::-1, :, s.argmax()]

    a = np.nonzero(coronal)
    firstx = a[0][0]
    lastx = a[0][len(a[0]) - 1]

    coronal_crop = coronal[firstx + crop_val:lastx - crop_val, :]
    edges_coronal = auto_canny(coronal_crop)
    dummy = np.float32([[1, 0, 4], [0, 1, 0]])
    rows, cols = edges_coronal.shape[:2]
    shifted = cv.warpAffine(edges_coronal, dummy, (cols, rows))
    shifted = shifted > 0
    shifted = shifted * 1
    edge_dummy = edges_coronal > 1
    edge_dummy = edge_dummy * 1
    remove_pix_c = shifted + edge_dummy > 1
    remove_pix_c = remove_pix_c * 1
    eroded_c = ndimage.binary_erosion(remove_pix_c, structure=np.ones((1, 2)))
    c_erodedsum = eroded_c.sum()

    sagittal_crop = sagittal[firstx + crop_val:lastx - crop_val, :]
    edges_sagittal = auto_canny(sagittal_crop)
    dummy = np.float32([[1, 0, 4], [0, 1, 0]])
    rows, cols = edges_sagittal.shape[:2]
    shifted = cv.warpAffine(edges_sagittal, dummy, (cols, rows))
    shifted = shifted > 0
    shifted = shifted * 1
    edge_dummy = edges_sagittal > 1
    edge_dummy = edge_dummy * 1
    remove_pix_s = shifted + edge_dummy > 1
    remove_pix_s = remove_pix_s * 1
    eroded_s = ndimage.binary_erosion(remove_pix_s, structure=np.ones((1, 2)))
    s_erodedsum = eroded_s.sum()

    labels_c = measure.label(eroded_c)
    props_c = measure.regionprops(labels_c)

    labels_s = measure.label(eroded_s)
    props_s = measure.regionprops(labels_s)

    if len(props_c) > 10 and len(props_s) > 10 and c_erodedsum > 20 or s_erodedsum > 20:
        flag_mask_hole = True
    else:
        flag_mask_hole = False

    if z < 370:
        flag_mask_size = True
    else:
        flag_mask_size = False

    if (c_erodedsum > 10 and s_erodedsum > 10) or (c_erodedsum > 25 or s_erodedsum > 25) or flag_mask_hole:
        log.warning('There is a data issue')
        decision = 'Issue detected, please verify'
    elif flag_mask_size:
        msg = 'series_missing'
        log.warning('There is a data issue ({})'.format(msg))
        decision = 'Issue detected, please verify'
    else:
        decision = ''
    if ds.summary.exists('metadata.json'):
        with open(ds.summary.path('metadata.json'), 'r') as jsonfile:
            metadata = json.load(jsonfile)
    metadata['anomaly_detection'] = decision

    with open(ds.summary.path('metadata.json'), 'w') as jsonfile:
        json.dump(metadata, jsonfile, sort_keys=True, indent=4, ensure_ascii=False)


from __future__ import print_function, unicode_literals

from PyInquirer import style_from_dict, Token, prompt
import json

def initialize_project():

    style = style_from_dict({
        Token.QuestionMark: '#E91E63 bold',
        Token.Selected: '#673AB7 bold',
        Token.Instruction: '',  # default
        Token.Answer: '#2196f3 bold',
        Token.Question: '',
    })

    questions = [
        {
            'type': 'input',
            'name': 'name',
            'message': 'Package name:',
            'default': 'qwe'
        },
        {
            'type': 'input',
            'name': 'version',
            'message': 'Version:',
            'default': '1.0.0',
        },
        {
            'type': 'input',
            'name': 'description',
            'message': 'Description:',
            'default': '',
        },
        {
            'type': 'input',
            'name': 'repository',
            'message': 'Repository:',
        },
        {
            'type': 'input',
            'name': 'author',
            'message': 'Author:',
        },
        {
            'type': 'input',
            'name': 'keywords',
            'message': 'Keywords:',
        },
        {
            'type': 'input',
            'name': 'license',
            'message': 'License:',
        },

    ]
    answers = prompt(questions, style=style)
    if answers:
        with open('app.json', 'w') as config:
            json.dump(answers, config, indent=4)


from setuptools import setup, find_packages

readme = open('README.txt', 'rb').read()

setup(name          = 'OWSLib',
      version       = '0.3.2beta',
      description   = 'OGC Web Service utility library',
      long_description = readme,
      license       = 'BSD',
      keywords      = 'gis ogc ows wfs wms sos csw capabilities metadata',
      author        = 'Sean Gillies',
      author_email  = 'sgillies@frii.com',
      maintainer        = 'Sean Gillies',
      maintainer_email  = 'sgillies@frii.com',
      url           = 'http://trac.gispython.org/lab/wiki/OwsLib',
      packages      = find_packages(),
      test_suite    = 'owslib.tests.test_suite',
      classifiers   = [
        'Development Status :: 3 - Alpha',
        'Intended Audience :: Developers',
        'Intended Audience :: Science/Research',
        'License :: OSI Approved :: BSD License',
        'Operating System :: OS Independent',
        'Programming Language :: Python',
        'Topic :: Scientific/Engineering :: GIS',
        ],
)


"""
RandomForest.py
Written by Dan Adler
Email: (daadler0309@gmail.com)
GitHub: https://github.com/dadler6/
Self-implementation of a random forest.
Package requirements:
numpy
pandas
DecisionTree.py (self implementation)
"""

# Imports
import numpy as np
import DecisionTree as DT
from collections import Counter

class RandomForest(object):
    """
    Abstract decision tree class.  Will use a linked-list-esque implementation where node is a "Node" class,
    and the node holds a cutoff, and then reference to another node.  Nodes can also hold a terminating value.
    Abstraction is made using the private (_) to not be able to implement outside.
    Parameters:
    self._samp_size: Size of sample to take for fitting tree
    self._num_trees: The number of trees to create
    self._num_features: The number of features to use per tree (defaults to None which uses all features)
    self._split_type: whether to use gini vs. gain ratio etc
    self._terminate: The termination criteria (pure/not)
    self._leaf_terminate: The number of samples in a leaf (not used if self._teriminate = pure)
    self._oob_flag: The flag to calculate the oob error
    self._trees: The list of trees
    self._cols_used: The columns used to fit a tree
    self._oob_errors: The oob error per tree
    Methods:
        Public
        ------
        Initialization: Initializes the class
        fit: Fit the x data and y data
        get_oob_error: Get the out of bag error
        predict: Predict the values for a new dataset
        get_trees: Get the list of decision trees
        get_cols_used: Get the columns used for each tree
        Private
        -------
        __check_feature_size: Checks to make sure feature size is less than proposed number of features
        __get_sample: Take a random sample of the x/y data to produce a train/out of bad set
        __get_tree: Creates a new classification decision tree
        __calculate_oob_error: Calculates the out of back error for a specific sample
    """

    def __init__(
            self,
            samp_size=0.5,
            num_trees=10,
            num_features=None,
            split_type='gini',
            terminate='leaf',
            leaf_terminate=1,
            oob=False
    ):
        """
        Initialize the RandomForest class.
        :param samp_size: The number of samples to put within each decision tree
        :param num_trees: The number of trees to make
        :param split_type: The criteria to split on
        :param terminate: The termination criteria
        :param leaf_terminate: The number of samples to put into a leaf
        :param oob: Whether to cross-validated using an out-of-bag sample
        """
        # Set parameters
        self._samp_size = samp_size
        self._num_trees = num_trees
        self._num_features = num_features
        self._split_type = split_type
        self._terminate = terminate
        self._leaf_terminate = leaf_terminate
        self._oob_flag = oob
        self._trees = []
        self._cols_used = []
        self._oob_errors = []

    def fit(self, x_data, y_data):
        """
        Fit (train) a Random Forest model to the data.
        :param x_data: The dataset to train the decision tree with
        :param y_data: The result vector we are classifying (target)
        """
        # handle the data
        x_data, y_data = DT.ClassificationDecisionTree.handle_data(x_data, y_data)
        # Check feature size
        if self._num_features is None:
            self._num_features = x_data.shape[1]
        # Make the number of trees determinate by self._num_trees
        for i in range(self._num_trees):
            # Get tree
            x_in, y_in, x_out, y_out = self.__get_sample(x_data, y_data)
            cdt = self.__get_tree(x_in, y_in)
            # Calculate oob if necessary
            if self._oob_flag:
                self._oob_errors.append(self.__calculate_oob_error(cdt, x_out, y_out))
            self._trees.append(cdt)

    def __check_feature_size(self, x_data):
        """
        Check to make sure the feature size is <= x_data. If self._num_features is none, will set
        self._num_features to x_data
        :param x_data: The x_data matrix to fit to
        """
        # Check if self._num_features is none, if so set to shape
        if self._num_features is None:
            self._num_features = x_data.shape[1]
        if self._num_features > x_data.shape[1]:
            raise ValueError('Number of features is greater than given X features.')

    def __get_sample(self, x, y):
        """
        Get a sample from two indices.  Will also sample num features if necessary
        :param x: The x data to sample from
        :param y: The y data to sample from
        :return: The sampled x data, y data and the out of sample x data/y data
        """
        # Take the random sample
        idx = np.random.choice(len(y), size=int(np.floor(self._samp_size * len(y))), replace=True)
        # Sample features
        cols = np.random.choice(x.shape[1], size=self._num_features, replace=False)
        self._cols_used.append(cols)

        # Get the sample splits
        mask = np.ones(len(y), dtype=bool)
        mask[idx] = False
        x_in = x[idx, :]
        x_in = x_in[:, cols]
        y_in = y[idx]
        x_out = x[mask, :]
        x_out = x_out[:, cols]
        y_out = y[mask]

        return x_in, y_in, x_out, y_out

    def __get_tree(self, x, y):
        """
        Create a decision tree based upon self._num_trees.
        :param x: The x data to fit to (input)
        :paray y: The y data to fit to (target)
        :return: A new CDT
        """
        dt = DT.ClassificationDecisionTree(self._split_type, self._terminate,  self._leaf_terminate, prune=False)
        dt.fit(x, y)
        return dt

    @staticmethod
    def __calculate_oob_error(cdt, x_out, y_out):
        """
        Calculate the oob error for a tree by predicting on the out of bag sample.
        :param cdt: The fit decision tree
        :param x_out: The out of bag input
        :param y_out: THe out of bad target
        """
        y_pred = cdt.predict(x_out)
        return (np.mean(y_out) - np.mean(y_pred))**2

    def get_oob_error(self):
        return np.mean(self._oob_errors)

    def predict(self, x_data):
        """
        Predict the y (target) for this x_data
        :param x_data: The data to predict off of
        :return: The predicted target data (y)
        """
        # Handle data
        x_data, _ = DT.ClassificationDecisionTree.handle_data(x_data)
        preds = []
        # Predict on each tree
        for i in range(len(self._trees)):
            preds.append(self._trees[i].predict(x_data[:, self._cols_used[i]]))
        pred = np.column_stack(preds)
        return np.array([Counter(pred[i, :]).most_common(1)[0][0] for i in range(pred.shape[0])])

    def get_trees(self):
        """
        Get the list of trees.
        :return: The list of trees.
        """
        return self._trees

    def get_cols_used(self):
        """
        Get the columns used for each tree.
        :return: The columns used
        """
        return self._cols_used


#O(nlogn)

def heap_sort(T):
    def heapify(T, n, l):
        largest = l
        left = 2*l + 1
        right = 2*l + 2

        if left < n and T[left] > T[largest]:
            largest = left
        if right < n and T[right] > T[largest]:
            largest = right
        if l != largest:
            T[l], T[largest] = T[largest], T[l]
            heapify(T, n, largest)

    n = len(T)

    for i in range(n//2 - 1, -1, -1):
        heapify(T, n, i)
    for i in range(n-1, 0, -1):
        T[i], T[0] = T[0], T[i]
        heapify(T, i, 0)

    return T


# description : This file contains the Rent class and a decorator. The rent class calculates
#               both the mean rental price for each year and the percent increase for each city
#               and creates a plot for both the mean rental price and percent increase.

import numpy as np
import csv
import matplotlib.pyplot as plt
import matplotlib

def showNums(func):
    """ it is a decorator, print out the return value of the fuction passed into """
    def returnMethod(*args, **kwargs):
        returnValue = func(*args, **kwargs)
        print("returnValue of function:", returnValue)
        return returnValue
    return returnMethod

class Rent:
    START_YEAR = 2011
    END_YEAR = 2018
    def __init__(self):
        try:
            filename = "lab2prices.csv"
            self._data = np.loadtxt(filename, usecols=(i for i in range(2, ((self.END_YEAR - self.START_YEAR + 1) * 12) + 1)), dtype= int, delimiter=',')
            filename = "lab2cities.txt"
            with open(filename) as infile:
                self._cityList = [ line.rstrip()      for line in infile       if not line.isspace()]

            self._arrMean = self.calcMeanRentalPrice()
            self._arrIncrease = self.calcPercentIncrease()
        except OSError:
            raise OSError(filename)
        except FileNotFoundError:
            raise FileNotFoundError(filename)

    def calcMeanRentalPrice(self):
        """ create an np array, assign the data into the np array, return the np array"""
        arrMean = np.zeros((  len(self._cityList)  ,  (self.END_YEAR - self.START_YEAR + 1) ))
        for index in range(0,(self.END_YEAR - self.START_YEAR + 1)*12,12):
            arrMean[:,index//12] = self._data[:, index:index+12].mean(1)

        return arrMean

    @showNums
    def calcPercentIncrease(self) :
        """ get the means of tne start year and the end year, store the percent increase into the np array, return the np array"""
        arrIncrease = ((self._arrMean[:,-1] - self._arrMean[:,0])/self._arrMean[:,0])*100
        return arrIncrease

    def plotRentalPrice(self, index) :
        """ plots the rental price by means of start year and end year,
            plots the graph depending on what the user chooses,
            plots All if the user chooses 'All' option """
        if index + 1 > len(self._cityList):
            colorTup = ('xkcd:blue', 'xkcd:aqua', 'xkcd:beige', 'xkcd:black', 'xkcd:teal', 'xkcd:chartreuse', 'xkcd:coral', 'xkcd:sienna', 'xkcd:olive', 'xkcd:fuchsia', 'xkcd:gold', 'xkcd:grey', 'xkcd:purple')
            for i in range(len(self._cityList)):
                plt.plot(range(1,(self.END_YEAR - self.START_YEAR + 1)+1), self._arrMean[i,:], colorTup[i], label = self._cityList[i])
        else:
            plt.plot(range(1,(self.END_YEAR - self.START_YEAR + 1)+1), self._arrMean[index,:], label = self._cityList[index])
        plt.xticks(range(1, (self.END_YEAR - self.START_YEAR + 1)+1), range(self.START_YEAR, self.END_YEAR + 1))
        plt.legend(loc = "best")
        plt.title("Rental price")
        plt.xlabel("Year")
        plt.ylabel("Price")

    def plotPercentIncrease(self):
        """ plots the percent increase with the names of the cities as the x-axis,
            and the percent increase as the y-axis """
        plt.plot(range(0, len(self._cityList)), self._arrIncrease, "k", marker = "o")
        plt.bar(range(0, len(self._cityList)), self._arrIncrease, align = "center")
        plt.title("Percent Increase between 2011 and 2018")
        plt.xlabel("Cities")
        plt.ylabel("Percent Increase(%)")
        plt.xticks(range(0,len(self._cityList)), self._cityList, rotation = "vertical")
        plt.ylim(min(self._arrIncrease)*0.50 , max(self._arrIncrease)*1.1)

    def getCityList(self):
        """ returns a list of city names stored in instance variable self._cityList """
        return self._cityList


from collections import defaultdict

class BallroomBayesNet():
    def __init__(self, m, n, debug = True):
        self.width = m
        self.height = n

        #dictionaries to store CPTs
        self.l_CPT = defaultdict(dict)
        self.c_CPT = defaultdict(dict)
        self.m1_CPT = defaultdict(dict)
        self.m2_CPT = defaultdict(dict)
        self.s_CPT = defaultdict(dict)
        self.possibleLocations = []
        self.DEBUG = debug

        for i in range(m):
            for j in range(n):
                self.possibleLocations.append((i,j))
        #initialize last location CPT and initialize current location CPT
        for location in self.possibleLocations:
            self.l_CPT[location] = (1.0 / (self.width*self.height))
            self._generate_c_prob(*location)
            self._generate_m1_prob(*location)
            self._generate_m2_prob(*location)
            self._generate_S_prob(*location)

        print("Initial distribution")
        self._print_last_location_dist()
        if self.DEBUG:
            self._print_current_location_dist()
            self._print_m1_dist()
            self._print_m2_dist()
            self._print_s_dist()

    def _print_last_location_dist(self):
        print("Last location distribution:")
        for key in self.l_CPT:
            print("\t Last location: " + str(key) + ", prob: " + str(self.l_CPT[key]))
        print('\n')

    def _print_current_location_dist(self):
        print("Current location distribution: ")
        for lastLoc in self.c_CPT:
            print("\tLast location: " + str(lastLoc))
            for currLoc in self.c_CPT[lastLoc]:
                if self.c_CPT[lastLoc][currLoc] != 0.0:
                    print("\t\tCurrent location: " + str(currLoc) + ", prob: " + str(self.c_CPT[lastLoc][currLoc]))

        print('\n')
    def _print_m1_dist(self):
        print("Motion sensor #1 distribution: ")
        for currLoc in self.m1_CPT:
            print("\tCurrent location: " + str(currLoc) + ", True prob: " + str(self.m1_CPT[currLoc][True])
            + ", False prob: " + str(self.m1_CPT[currLoc][False]))
        print('\n')

    def _print_m2_dist(self):
        print("Motion sensor #2 distribution: ")
        for currLoc in self.m1_CPT:
            print("\tCurrent location: " + str(currLoc) + ", True prob: " + str(self.m2_CPT[currLoc][True])
            + ", False prob: " + str(self.m2_CPT[currLoc][False]))
        print('\n')

    def _print_s_dist(self):
        print("Sound sensor distribution: ")
        for currLoc in self.s_CPT:
            print("\tCurrent location: " + str(currLoc))
            for sLoc in self.s_CPT[currLoc]:
                if self.s_CPT[currLoc][sLoc] != 0.0:
                    print("\t\t Sound reported at: " + str(sLoc) + ", prob: " + str(self.s_CPT[currLoc][sLoc]))

    def _generate_list_of_locations_one_step_away(self,x,y):
        output = []
        if x - 1 >= 0:
            output.append((x-1, y))
        if x + 1 < self.width:
            output.append((x+1, y))

        if y - 1 >= 0:
            output.append((x, y-1))
        if y + 1 < self.height:
            output.append((x, y+1))

        return output

    def _generate_list_of_locations_two_steps_away(self, x,y):
        output = []
        if x - 2 >= 0:
            output.append((x-2, y))
        if x + 2 < self.width:
            output.append((x+2, y))

        if y - 2 >= 0:
            output.append((x, y-2))
        if y + 2 < self.height:
            output.append((x, y+2))

        toCheck = [(1,1), (-1,1), (1,-1), (-1,-1)]
        for i in toCheck:
            if x + i[0] >= 0 and x + i[0] < self.width:
                if y + i[1] >= 0 and y + i[1] < self.height:
                    output.append((x + i[0], y + i[1]))

        return output

     #generates probabilities for C given L (x,y)
    def _generate_c_prob(self, x, y):
        possibleNextLocations = self._generate_list_of_locations_one_step_away(x,y)
        for loc2 in possibleNextLocations:
            self.c_CPT[(x,y)][loc2] = (1.0 / len(possibleNextLocations))
        #set prob 0 for all tiles that can't be reached
        for loc2 in list(set(self.possibleLocations) - set(possibleNextLocations)):
            self.c_CPT[(x,y)][loc2] = 0.0

    #generates probability of M1=True given C (x,y)
    def _generate_m1_prob(self, x, y):
        #if monkey isn't in m1 line of sight, prob is 0.05
        if x != 0 and y != 0:
            self.m1_CPT[(x,y)][True] = 0.05
            self.m1_CPT[(x,y)][False] = 0.95
        else:
            if y == 0:
                val = 0.9 - (0.1*x)
                self.m1_CPT[(x,y)][True] = val
                self.m1_CPT[(x,y)][False] = 1.0 - val
            elif x == 0:
                val = 0.9 - (0.1*y)
                self.m1_CPT[(x,y)][True] = val
                self.m1_CPT[(x,y)][False] = 1.0 - val

    #generates probability of M2=True given C (x,y)
    def _generate_m2_prob(self, x, y):
        if x != self.width - 1 and y != self.height - 1:
            self.m2_CPT[(x,y)][True] = 0.05
            self.m2_CPT[(x,y)][False] = 0.95
        else:
            if y == self.height - 1:
                dist = (self.width-1) - x
                val = 0.9 - (0.1*dist)
                self.m2_CPT[(x,y)][True] = val
                self.m2_CPT[(x,y)][False] = 1.0 - val
            elif x == self.width - 1:
                dist = (self.height-1) - y
                val = 0.9 - (0.1*dist)
                self.m2_CPT[(x,y)][True] = val
                self.m2_CPT[(x,y)][False] = 1.0 - val

    #generates probabilities for S given C (x,y)
    def _generate_S_prob(self, x, y):

        locationsOneAway = self._generate_list_of_locations_one_step_away(x,y)

        for loc in locationsOneAway:
            self.s_CPT[(x,y)][loc] = (0.30 / len(locationsOneAway))

        locationsTwoAway = self._generate_list_of_locations_two_steps_away(x,y)

        for loc in locationsTwoAway:
            self.s_CPT[(x,y)][loc] = (0.10 / len(locationsTwoAway))

        #assign all other possible locations not one or two away a prob of 0
        others = list(set(self.possibleLocations) - set(locationsOneAway) - set(locationsTwoAway))
        for loc in others:
            self.s_CPT[x,y][loc] = 0.0
        self.s_CPT[(x,y)][(x,y)] = 0.6
        #print(self.s_CPT)

    #calculates (un-normalized) probability for a given C (x,y) given m1, m2, and s
    def _calculate_prob_of_C(self, c_x, c_y, m1: bool, m2:bool, s_x, s_y):
        #sum over all possible last locations
        prob = 0.0
        if self.DEBUG:
            print("Calculating total probability for current location (" + str(c_x) + ", " + str(c_y) + ")" )

        for l in self.possibleLocations:
            if self.DEBUG:
                print("\tEvaluating for last location " + str(l))
                print("\t \t Multiplying: " + str(self.l_CPT[l]) + " " + str(self.c_CPT[l][(c_x, c_y)]) + " " +
                str(self.m1_CPT[(c_x, c_y)][m1]) + " " + str(self.m2_CPT[(c_x, c_y)][m2]) + " " +
                str(self.s_CPT[(c_x, c_y)][(s_x, s_y)]))

            prob += (self.l_CPT[l] * self.c_CPT[l][(c_x, c_y)] * self.m1_CPT[(c_x, c_y)][m1]
                    * self.m2_CPT[(c_x, c_y)][m2] * self.s_CPT[(c_x, c_y)][(s_x, s_y)])

        return prob

    #generates prob distribution over all possible values of C given values of m1, m2, and s
    def get_distrib_over_all_C(self, m1: bool, m2:bool, s_x, s_y):
        #dict to store un-normalized values to be normalized and stored in self.c_CPT
        unnormalized = defaultdict(dict)

        #variable to store sum of un-normalized probabilities
        sum_of_unnormalized = 0.0

        #loop over all values on the board
        for loc in self.possibleLocations:
            unnormalized[loc] = self._calculate_prob_of_C(*loc, m1, m2, s_x, s_y)
            sum_of_unnormalized += unnormalized[loc]

        #dictionary to store normalized probabilities
        normalized = defaultdict(dict)

        #normalization
        for loc in unnormalized:
            normalized[loc] = (unnormalized[loc] / sum_of_unnormalized)

        if self.DEBUG:
            print("BEFORE NORMALIZATION:")
            for loc in unnormalized:
                print("\t Location: " + str(loc) + ", prob: " + str(unnormalized[loc]))
            print("AFTER NORMALIZATION:")

        for loc in normalized:
            print("\t Location: " + str(loc) + ", prob: " + str(normalized[loc]))

        #then, we set l_cpt equal to normalized
        self.l_CPT = normalized


import pyfdb
import shutil

fdb = pyfdb.FDB()

### Archive ###
key = {
    'domain': 'g',
    'stream': 'oper',
    'levtype': 'pl',
    'levelist': '300',
    'date': '20191110',
    'time': '0000',
    'step': '0',
    'param': '138',
    'class': 'rd',
    'type': 'an',
    'expver': 'xxxx'
}

filename = 'x138-300.grib'
fdb.archive(open(filename, "rb").read(), key)

key['levelist'] = '400'
filename = 'x138-400.grib'
pyfdb.archive(open(filename, "rb").read())

key['expver'] = 'xxxy'
filename = 'y138-400.grib'
fdb.archive(open(filename, "rb").read())
fdb.flush()

### List ###
request = {
    'class': 'rd',
    'expver': 'xxxx',
    'stream': 'oper',
    'date': '20191110',
    'time': '0000',
    'domain': 'g',
    'type': 'an',
    'levtype': 'pl',
    'step': 0,
    'levelist': [300, '500'],
    'param': ['138', 155, 't']
}
print('direct function, request as dictionary:', request)
for el in pyfdb.list(request, True):
    assert(el['path'])
    assert(el['path'].find('rd:xxxx:oper:20191110:0000:g/an:pl.') != -1)
    assert(not 'keys' in el)

request['levelist'] = ['100', '200', '300', '400', '500', '700', '850', '1000']
request['param'] = '138'
print('')
print('direct function, updated dictionary:', request)
it = iter(pyfdb.list(request, True, True))

el = next(it)
assert(el['path'])
assert(el['path'].find('rd:xxxx:oper:20191110:0000:g/an:pl.') != -1)
assert(el['keys'])
keys = el['keys']
assert(keys['class'] == 'rd')
assert(keys['levelist'] == '300')

el = next(it)
assert(el['path'])
assert(el['path'].find('rd:xxxx:oper:20191110:0000:g/an:pl.') != -1)
assert(el['keys'])
keys = el['keys']
assert(keys['class'] == 'rd')
assert(keys['levelist'] == '400')

try:
    el = next(it)
    assert False, "returned unexpected field"
except StopIteration:
    assert True, "field listing completed"

# as an alternative, create a FDB instance and start queries from there
request['levelist'] = ['400', '500', '700', '850', '1000']
print('')
print('fdb object, request as dictionary:', request)
for el in fdb.list(request, True, True):
    assert(el['path'])
    assert(el['path'].find('rd:xxxx:oper:20191110:0000:g/an:pl.') != -1)
    assert(el['keys'])
    keys = el['keys']
    assert(keys['class'] == 'rd')
    assert(keys['levelist'] == '400')

### Retrieve ###
request = {
    'domain': 'g',
    'stream': 'oper',
    'levtype': 'pl',
    'step': '0',
    'expver': 'xxxx',
    'date': '20191110',
    'class': 'rd',
    'levelist': '300',
    'param': '138',
    'time': '0000',
    'type': 'an'
}

filename = 'x138-300bis.grib'
print('')
print('save to file ', filename)
with open(filename, 'wb') as o, fdb.retrieve(request) as i:
    shutil.copyfileobj(i, o)

request['levelist'] = '400'
filename = 'x138-400bis.grib'
print('save to file ', filename)
with open(filename, 'wb') as o, fdb.retrieve(request) as i:
    shutil.copyfileobj(i, o)

request['expver'] = 'xxxy'
filename = 'y138-400bis.grib'
print('save to file ', filename)
with open(filename, 'wb') as o, pyfdb.retrieve(request) as i:
    shutil.copyfileobj(i, o)

# request = {
#     'class': 'od',
#     'expver': '0001',
#     'stream': 'oper',
#     'date': '20040118',
#     'time': '0000',
#     'domain': 'g',
#     'type': 'an',
#     'levtype': 'sfc',
#     'step': 0,
#     'param': 151
# }
print('')
print('FDB retrieve')
print('direct function, retrieve from request:', request)
datareader = pyfdb.retrieve(request)

print('')
print('reading a small chunk')
chunk = datareader.read(10)
print(chunk)
print('tell()', datareader.tell())

print('go back (partially) - seek(2)')
datareader.seek(2)
print('tell()', datareader.tell())

print('reading a larger chunk')
chunk = datareader.read(40)
print(chunk)

print('go back - seek(0)')
datareader.seek(0)

print('')
print('decode GRIB')
from pyeccodes import Reader
reader = Reader(datareader)
grib = next(reader)
grib.dump()

request['levelist'] = [300, '400']
request['expver'] = 'xxxx'
filename = 'foo.grib'

print('')
print('save to file ', filename)
with open(filename, 'wb') as o, fdb.retrieve(request) as i:
    shutil.copyfileobj(i, o)


"""
1231. Divide Chocolate
You have one chocolate bar that consists of some chunks. Each chunk has its own sweetness given by the array sweetness.

You want to share the chocolate with you k friends so you start cutting the chocolate bar into k + 1 pieces using k cuts, each piece consists of some consecutive chunks.

Being generous, you will eat the piece with the minimum total sweetness and given the other pieces to your friends.

Find the maximum total sweetness of the piece you can get by cutting the chocolate bar optimally.

Example1:
Input: sweetness = [1,2,3,4,5,6,7,8,9], k = 5
Output: 6
Explanation: You can divide the chocolate to [1,2,3], [4,5], [6], [7], [8], [9]

Example2:
Input: sweetness = [5,6,7,8,9,1,2,3,4], k = 8
Output: 1
Explanation: There is only one way to cut the bar into 9 pieces.

Example3:
Input: sweetness = [1,2,2,1,2,2,1,2,2], k = 2
Output: 5
Explanation: You can divide the chocolate to [1,2,2], [1,2,2], [1,2,2]

Constraints:
0 <= k < sweetness.length <= 10^4
1 <= sweetness[i] <= 10^5
"""

"""
Note:
1. Binary Search: O(nlogn) time | O(1) space
"""

from typing import List
class Solution:
    def maximizeSweetness(self, sweetness: List[int], k: int) -> int:
        def condition(sweet) -> bool:
            currentSum = currentK = 0
            for s in sweetness:
                currentSum += s
                if currentSum >= sweet:
                    currentSum = 0
                    currentK += 1
                if currentK >= k+1:
                    return True
            return currentK >= k+1

        left, right = min(sweetness), sum(sweetness) + 1
        while left < right:
            mid = left + (right - left) // 2
            if condition(mid):
                left = mid + 1
            else:
                right = mid
        return left - 1

# Unit Tests
import unittest
funcs = [Solution().maximizeSweetness]
class TestMaximizeSweetness(unittest.TestCase):
    def testMaximizeSweetness1(self):
        for func in funcs:
            sweetness = [1,2,3,4,5,6,7,8,9]
            k = 5
            self.assertEqual(func(sweetness=sweetness, k=k), 6)

    def testMaximizeSweetness2(self):
        for func in funcs:
            sweetness = [5,6,7,8,9,1,2,3,4]
            k = 8
            self.assertEqual(func(sweetness=sweetness, k=k), 1)

    def testMaximizeSweetness3(self):
        for func in funcs:
            sweetness = [1,2,2,1,2,2,1,2,2]
            k = 2
            self.assertEqual(func(sweetness=sweetness, k=k), 5)

if __name__ == "__main__":
    unittest.main()


import fileinput

def fib_period(m):
    f0 = 0
    f1 = 1
    period = [0, 1]
    for _ in range(2, 10**5):
        f0, f1 = f1%m, (f0+f1)%m
        period.append(f1)
        if f0 == 0 and f1 == 1:
            return period[:-2]

finput = fileinput.input()
m, n = next(finput).split()
m, n = int(m), int(n)

p = fib_period(10)
p_sum = sum(p)
lp = len(p)

quotient = (m-1) // lp
remainder = (m-1) % lp
sm = p_sum*quotient + sum(p[:remainder+1])

quotient = n // lp
remainder = n % lp
sn = p_sum*quotient + sum(p[:remainder+1])
#print(f'Sn={sn} Sm={sm}')
print((sn-sm)%10)


import pytest
from click.testing import CliRunner
from cicdctl import cli

allowed_networks_outputs = [
    ('list', [], ''),
    ('add', ['--cidr', 'my-ip'], ''),
    ('add', ['--cidr', '67.180.81.198/32'], ''),
    ('remove', ['--cidr', 'my-ip'], ''),
    ('remove', ['--cidr', '67.180.81.198/32'], ''),
]

@pytest.mark.parametrize('command,opts,output', allowed_networks_outputs)
def test_command(runner, command, opts, output):
    result = runner.invoke(cli, ['--dry-run', 'allowed-networks', command] + opts)
    assert result.exit_code == 0
    #assert output in result.output


import matplotlib.pyplot as plt
import tensorflow as tf
import pandas as pd
import numpy as np

# 设置数据
(train_image, train_label),(test_image, test_label) = tf.keras.datasets.fashion_mnist.load_data()
train_label_onehot = tf.keras.utils.to_categorical(train_label)
test_label_onehot = tf.keras.utils.to_categorical(test_label)

# 搭建模型
model = tf.keras.Sequential()
model.add(tf.keras.layers.Flatten(input_shape=(28, 28)))
model.add(tf.keras.layers.Dense(128, activation=tf.keras.activations.relu))
# 添加一个dropout层
model.add(tf.keras.layers.Dropout(0.5)) # 参数0.5：每次随机丢弃，一半神经元
model.add(tf.keras.layers.Dense(128, activation=tf.keras.activations.relu))
model.add(tf.keras.layers.Dropout(0.5)) # 参数0.5：每次随机丢弃，一半神经元
model.add(tf.keras.layers.Dense(128, activation=tf.keras.activations.relu))
model.add(tf.keras.layers.Dropout(0.5)) # 参数0.5：每次随机丢弃，一半神经元
model.add(tf.keras.layers.Dense(10, activation=tf.keras.activations.softmax))

# 打印模型
model.summary()

# 模型训练
model.compile(optimizer=tf.keras.optimizers.Adam(),
              loss=tf.keras.losses.categorical_crossentropy,
              metrics=['acc'])
history = model.fit(train_image, train_label_onehot, epochs=10,
                    # validation_data参数用途：添加验证数据，每个epoch训练完后，评估这个model的正确率
                    validation_data=(test_image, test_label_onehot))

# 打印history字典中，有哪些关键词
print(history.history.keys())

plt.plot(history.epoch, history.history.get('loss', 'can not find the key'), label='loss')
plt.plot(history.epoch, history.history.get('val_loss', 'can not find the key'), label='val_loss')
plt.legend()    # 设置标签最佳位置
plt.show()


import re
import xml.etree.ElementTree as ET
from impacket.ldap import ldap, ldapasn1
from impacket.smbconnection import SessionError
from impacket.smb3structs import FILE_READ_DATA, FILE_WRITE_DATA

from lib.adscan.accesscontrol import parse_accesscontrol, parse_sd, process_sid

class GPO:

    name_to_sid = {
        'administrators': 'S-1-5-32-544',
        'remote desktop users': 'S-1-5-32-555',
        'distributed com users': 'S-1-5-32-562',
        'remote management users': 'S-1-5-32-580',
    }

    @classmethod
    def list(self, ldap_obj, callback):

        schema_guid_dict = ldap_obj._get_schema_guid_dict(['Group-Policy-Container', 'ms-mcs-admpwd', 'ms-DS-Key-Credential-Link', 'Service-Principal-Name'])

        def process(item):
            if isinstance(item, ldapasn1.SearchResultEntry) is not True:
                return

            attr = ldap_obj.to_dict_impacket(item)

            domain = ".".join([item.split("=", 1)[-1] for item in str(attr['distinguishedName']).split(',') if item.split("=",1)[0].lower() == "dc"])
            domain_dn = ",".join(["DC=%s" % p for p in domain.split('.')])
            domain_sid = ldap_obj.resolve_dn_to_sid([domain_dn])[0]

            dn = str(attr['distinguishedName'])

            b = bytes(attr['objectGUID'])
            guid = b[0:4][::-1].hex() + '-'
            guid += b[4:6][::-1].hex() + '-'
            guid += b[6:8][::-1].hex() + '-'
            guid += b[8:10].hex() + '-'
            guid += b[10:16].hex()

            aces = parse_sd(bytes(attr['nTSecurityDescriptor']), domain.upper(), 'group-policy-container', schema_guid_dict)

            callback({
                'domain': domain,
                'domain_sid': domain_sid,
                'name': str(attr['displayName']),
                'dn': dn,
                'guid': guid,
                'gpcpath': str(attr['gPCFileSysPath']),
                'aces': aces,
            })

        sc = ldap.SimplePagedResultsControl(size=100)
        sc2 = ldapasn1.SDFlagsControl(criticality=True, flags=0x7)
        attributes = ['name', 'displayName', 'distinguishedName', 'objectGUID', 'nTSecurityDescriptor', 'gPCFileSysPath']
        sbase = "%s" % ldap_obj.defaultdomainnamingcontext
        ldap_obj.conn.search(searchBase=sbase, searchFilter='(objectCategory=groupPolicyContainer)', searchControls=[sc, sc2], perRecordCallback=process, attributes=attributes)

    @classmethod
    def resolve_effect(self, smbscan, ldap_obj, gpo_dn, gpo_path, gpo_effect):

        gpo_domain = ".".join([item.split("=", 1)[-1] for item in str(gpo_dn).split(',') if item.split("=",1)[0].lower() == "dc"])

        share_pattern = re.compile("\\\\\\\\([^\\\\]+)\\\\([^\\\\]+)(\\\\.*)")
        m = share_pattern.match(gpo_path)
        if m:
            tid = smbscan.conn.connectTree(m.group(2))

            try:
                file_path = m.group(3) + "\\" + '\\'.join(["MACHINE", "Microsoft", "Windows NT", "SecEdit", "GptTmpl.inf"])
                #print(file_path)
                fid = smbscan.conn.openFile(tid, file_path, desiredAccess=FILE_READ_DATA)
                file_data = smbscan.conn.readFile(tid, fid)
                smbscan.conn.closeFile(tid, fid)
            except SessionError:
                file_data = None

            if file_data != None:
                try:
                    file_data = file_data.decode('utf-8')
                except UnicodeDecodeError as e:
                    file_data = file_data.decode('utf-16')

                #print("================\n%s\n================" % file_data)

                group_membership = False
                for line in file_data.split('\n'):
                    line = line.strip()

                    if len(line) == 0:
                        continue

                    if line.startswith('['):
                        if line.startswith('[Group Membership]'):
                            group_membership = True
                        else:
                            group_membership = False
                        continue

                    if group_membership:
                        left = line.split("=")[0].strip()
                        right = line.split("=")[-1].strip()
                        if len(right) != 0:
                            right = right.split(',')
                        else:
                            right = []

                        action_type = left.split("__")[-1]
                        left = left.split("__")[0]

                        #print("####################")
                        #print(action_type)
                        #print(left)
                        #print(right)

                        from lib.adscan.ou import OU
                        if action_type == "Members":
                            if left.startswith('*'):
                                left = left[1:]

                            if left in OU.privileged_sid_dict:
                                members = []
                                for sid in right:
                                    if sid.startswith('*'):
                                        sid = sid[1:]

                                    if not sid.startswith('S-'):
                                        # to resolve
                                        sid = ldap_obj._resolve_name_to_sid(gpo_domain, sid)

                                    if sid != None and sid.startswith('S-') and len(sid.split('-')) == 8:
                                        # Domain SID
                                        members.append({
                                            'ObjectIdentifier': sid,
                                        })

                                # This replaces all previous entries
                                gpo_effect[left]["Members"] = members
                        elif action_type == "Memberof":
                            if left.startswith('*'):
                                left = left[1:]

                            if not left.startswith('S-'):
                                # to resolve
                                left = ldap_obj._resolve_name_to_sid(gpo_domain, left)

                            if left != None and left.startswith('S-') and len(left.split('-')) == 8:
                                # Domain SID
                                for sid in right:
                                    if sid.startswith('*'):
                                        sid = sid[1:]

                                    if sid in OU.privileged_sid_dict:
                                        gpo_effect[sid]["Memberof"].append({
                                            'ObjectIdentifier': left,
                                        })

            try:
                file_path = m.group(3) + "\\" + '\\'.join(["MACHINE", "Preferences", "Groups", "Groups.xml"])
                fid = smbscan.conn.openFile(tid, file_path, desiredAccess=FILE_READ_DATA)
                file_data = smbscan.conn.readFile(tid, fid)
                smbscan.conn.closeFile(tid, fid)
            except SessionError:
                file_data = None

            if file_data != None:
                file_data = file_data.decode()
                #print("================\n%s\n================" % file_data)

                root = ET.fromstring(file_data)

                if root.tag == "Groups":
                    for group in root:
                        if group.tag != "Group":
                            continue

                        for prop in group:
                            if prop.tag != "Properties":
                                continue

                            action = prop.attrib['action']
                            if action != "U":
                                continue

                            groupSid = prop.attrib['groupSid'] if 'groupSid' in prop.attrib else None
                            groupName = prop.attrib['groupName'] if 'groupName' in prop.attrib else None

                            if not groupSid:
                                if groupName:
                                    if groupName.lower() in self.name_to_sid:
                                        groupSid = self.name_to_sid[groupName.lower()]

                            from lib.adscan.ou import OU
                            if groupSid in OU.privileged_sid_dict:
                                if prop.attrib['deleteAllUsers'] == "1":
                                    for entry in gpo_effect[groupSid]['Localgroup']:
                                        if entry['ObjectType'] == 'User':
                                            del gpo_effect[groupSid]['Localgroup'][entry]

                                if prop.attrib['deleteAllGroups'] == "1":
                                    for entry in gpo_effect[groupSid]['Localgroup']:
                                        if entry['ObjectType'] == 'Group':
                                            del gpo_effect[groupSid]['Localgroup'][entry]

                                for members in prop:
                                    if members.tag != "Members":
                                        continue

                                    for member in members:
                                        action = member.attrib['action']

                                        memberSid = member.attrib['sid'] if 'sid' in member.attrib else None
                                        memberName = member.attrib['name'] if 'name' in member.attrib else None

                                        if not memberSid:
                                            memberSid = ldap_obj._resolve_name_to_sid(gpo_domain, memberName)

                                        if memberSid:
                                            if action.lower() == "add":
                                                gpo_effect[groupSid]['Localgroup'].append({
                                                    'ObjectIdentifier': memberSid,
                                                })

                                            elif action.lower() == "delete":
                                                for entry in gpo_effect[groupSid]['Localgroup']:
                                                    if entry['ObjectIdentifier'] == memberSid:
                                                        del gpo_effect[groupSid]['Localgroup'][entry]

        return ldap_obj._resolve_sid_types(gpo_effect, 'gpo_effect')

    @classmethod
    def merge_gpo_effect(self, gpo_effect):

        from lib.adscan.ou import OU
        out = {}
        for localgroup_sid, item in gpo_effect.items():
            localgroup_key = OU.privileged_sid_dict[localgroup_sid]

            entries = item["Memberof"]
            if len(item["Members"]) != 0:
                entries += item["Members"]
            else:
                entries += item["Localgroup"]

            out[localgroup_key] = entries

        return out


"""empty message

Revision ID: 47e94ae22418
Revises: 1e3e84a18f07
Create Date: 2019-08-10 07:51:50.274272

"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql

# revision identifiers, used by Alembic.
revision = '47e94ae22418'
down_revision = '1e3e84a18f07'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('events',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('title', sa.String(length=120), nullable=False),
    sa.Column('location', sa.String(length=255), nullable=False),
    sa.Column('manpower_quota', sa.Integer(), nullable=False),
    sa.Column('date_created', sa.DateTime(), nullable=False),
    sa.PrimaryKeyConstraint('id'),
    sa.UniqueConstraint('location'),
    sa.UniqueConstraint('title')
    )
    op.drop_table('event')
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('event',
    sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False),
    sa.Column('title', sa.VARCHAR(length=120), autoincrement=False, nullable=False),
    sa.Column('location', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
    sa.Column('manpower_quota', sa.INTEGER(), autoincrement=False, nullable=False),
    sa.Column('date_created', postgresql.TIMESTAMP(), autoincrement=False, nullable=False),
    sa.PrimaryKeyConstraint('id', name='event_pkey'),
    sa.UniqueConstraint('location', name='event_location_key'),
    sa.UniqueConstraint('title', name='event_title_key')
    )
    op.drop_table('events')
    # ### end Alembic commands ###


# -*- coding: utf-8 -*-
# @Author: Zeyuan Shang
# @Date:   2016-04-11 22:58:12
# @Last Modified by:   Zeyuan Shang
# @Last Modified time: 2016-04-11 22:58:18
from datetime import datetime

if __name__ == "__main__":
    T = int(input())
    pattern = '%a %d %b %Y %H:%M:%S %z'
    for _ in range(T):
        s = datetime.strptime(input(), pattern).timestamp()
        t = datetime.strptime(input(), pattern).timestamp()
        print(int(abs(s - t)))