# wrapper for observation preprocessing - features extraction from the game
import math, sys
from lux.game import Game
from lux.game_map import Cell, RESOURCE_TYPES
from lux.constants import Constants
from lux.game_constants import GAME_CONSTANTS
from lux import annotate
from kaggle_environments import make

import torch
import numpy as np

class EnvWrapper:
    def __init__(self, opponent):
        self.game_state = Game()
        self.opponent = opponent
        self.env = make("lux_ai_2021", configuration={"seed": 562124210, "loglevel": 0, "annotations": False}, debug=False)
        self.trainer = self.env.train([None,opponent])
        obs = self.trainer.reset()
        self.update_game_state(obs)
        self.device = "cuda" if (torch.cuda.is_available()) else "cpu"
        self.current_reward = 0

    def update_game_state(self, observation):
        """Update the local game state from observation"""

        if observation["step"] == 0:
            self.game_state = Game()
            self.game_state._initialize(observation["updates"])
            self.game_state._update(observation["updates"][2:])
            self.game_state.id = observation.player
        else:
            self.game_state._update(observation["updates"])
        self.obs = observation
        self.game_map = self.game_state.map
        self.player = self.game_state.players[observation.player]
        self.opponent = self.game_state.players[(observation.player + 1) % 2]
        self.width, self.height = self.game_map.width, self.game_map.height

        self.resource_tiles: list[Cell] = []
        self.city_tiles: list[Cell] = []

        for y in range(self.height):
            for x in range(self.width):
                cell = self.game_state.map.get_cell(x, y)
                if cell.has_resource():
                    self.resource_tiles.append(cell)
                if cell.citytile is not None:
                    self.city_tiles.append(cell.citytile)

        return self.game_state

    def get_reward(self):
        """Workaround to get a correct reward"""
        player = self.player
        ct_count = sum([len(v.citytiles) for k, v in player.cities.items()])
        unit_count = len(self.game_state.players[player.team].units)
        score = ct_count * 10000 + unit_count*10
        if score == self.current_reward:
            reward = 0
        if score > self.current_reward:
            reward = 1
        if score < self.current_reward:
            reward = -1
        self.current_reward = score

        return reward

    def get_cell_resources(self, cell: Cell):
        wood = 0
        coal = 0
        uranium = 0
        if cell.has_resource():
            wood = cell.resource.amount if cell.resource.type == "wood" else 0
            coal = cell.resource.amount if cell.resource.type == 'coal' else 0
            uranium = cell.resource.amount if cell.resource.type == 'uranium' else 0
        return wood, coal, uranium

    def get_adjacent_resources(self, pos):
        wood = 0
        coal = 0
        uranium = 0
        x, y = (pos.x, pos.y)
        for i in range(-1,1):
            for j in range(-1,1):
                cell = self.game_map.get_cell(x+i,y+j)
                if cell.has_resource():
                    wood += cell.resource.amount if cell.resource.type == "wood" else 0
                    coal += cell.resource.amount if cell.resource.type == 'coal' else 0
                    uranium += cell.resource.amount if cell.resource.type == 'uranium' else 0
        return wood, coal, uranium

    def add_units_state(self, state):
        # player units features
        for unit in self.player.units:
            active = 1 if unit.can_act() else 0.1
            cell = self.game_map.get_cell_by_pos(unit.pos)
            worker_type = 1 if unit.is_worker() else 0.5
            cargo = unit.get_cargo_space_left()
            x = unit.pos.x
            y = unit.pos.y
            is_in_city = cell.citytile != None
            adjacent_wood, adjacent_coal, adjacent_uranium = self.get_adjacent_resources(unit.pos)
            state[-8:-1, x, y] = torch.tensor([worker_type, cargo, adjacent_wood, adjacent_coal, adjacent_uranium, is_in_city, active]).to(self.device)
        # opponent units features
        for unit in self.opponent.units:
            x = unit.pos.x
            y = unit.pos.y
            cell_code = 1
            if unit.is_cart:
                cell_code +=0.5
            state[-8, x, y]= cell_code
        return state

    def get_global_state(self):
        state = torch.zeros(16,32,32, dtype=torch.float32).to(self.device)
        for x in range(self.width):
            for y in range(self.height):
                cell = self.game_map.get_cell(x,y)
                w, c, u = self.get_cell_resources(cell)
                cityid = int(cell.citytile.cityid[2:])+1 if cell.citytile else 0
                city = (cell.citytile.team+1) if cell.citytile else 0
                fuel = (self.opponent.cities[cell.citytile.cityid].fuel if city == self.opponent.team+1
                       else self.player.cities[cell.citytile.cityid].fuel if city == self.player.team+1 else 0)
                road = cell.road
                research = self.player.research_points
                state[:,x,y]= torch.tensor([w, c, u, cityid, fuel, city, road, research, 0, 0, 0, 0, 0, 0, 0, 0]).to(self.device)
        state = self.add_units_state(state)
        return state

    def step(self, action):
        obs, _, done, info = self.trainer.step(action)
        game_state = self.update_game_state(obs)
        reward = self.get_reward()
        global_state = self.get_global_state()
        return global_state, reward, done, info

    def reset(self):
        map_seed = np.random.randint(0,1000000000)
        self.env = make("lux_ai_2021", configuration={"seed": map_seed, "loglevel": 0, "annotations": False}, debug=False)
        self.trainer = self.env.train([None,self.opponent])
        obs = self.trainer.reset()
        self.update_game_state(obs)
        global_state = self.get_global_state()
        print("map size: ", self.height)
        return global_state

    def get_game_objects(self):
        return [self.player, self.city_tiles]


from neo4j.v1 import GraphDatabase

driver = GraphDatabase.driver("bolt://localhost", auth=("neo4j", "neo"))

with driver.session() as session:
    #     result = session.run("""\
    #     CALL db.propertyKeys()
    #     YIELD propertyKey
    #     WHERE propertyKey STARTS WITH "embedding_" AND not(propertyKey CONTAINS "lambda")
    #     RETURN propertyKey
    #     """)

    # property_keys = [row["propertyKey"] for row in result]
    property_keys = ["embedding_lpa"]

    # f1 f2 f3 f4 f5 .. fn label

    for property_key in property_keys:
        with open("emails/{0}-emails.txt".format(property_key), "w") as property_key_file:
            print("Property Key: ", property_key)
            result = session.run("""\
            // MATCH (enzyme:Person)
            // RETURN enzyme[$propertyKey] AS embedding, enzyme.label AS label
            MATCH (n:Node)
            WITH n.label as class, count(*) AS c
            ORDER BY c DESC
            WITH class WHERE c > 50
            WITH class ORDER BY class
            with collect(class) AS biggestClasses
            MATCH (p:Node) WHERE p.label IN biggestClasses
            RETURN p.`embedding-python` AS embedding, apoc.coll.indexOf(biggestClasses, p.label) AS label, p.label as initialLabel
            ORDER BY label
            """, {"propertyKey": property_key})

            for row in result:
                property_key_file.write(
                    "{0} {1}\n".format(" ".join([str(item) for item in row["embedding"]]), row["label"]))


import pytest
from httpx_backoff.backoff_options import Fibo

class TestBackoffFiboOption:
    @pytest.mark.parametrize(
        "expected",
        [[1, 1, 2, 3, 5, 8, 13], [1, 1, 2, 3, 5, 8, 13, 21, 34]],
    )
    def test_backoff_fibo_with_default_args(self, expected):
        fibo_gen = Fibo()

        for expect in expected:
            assert expect == next(fibo_gen)

    @pytest.mark.parametrize(
        "max_value,expected",
        [(8, [1, 1, 2, 3, 5, 8, 8, 8]), (9, [1, 1, 2, 3, 5, 8, 9]), (16, [1, 1, 2, 3, 5, 8, 13, 16])],
    )
    def test_backoff_fibo_with_max_value(self, max_value, expected):
        fibo_gen = Fibo(max_value=max_value)

        for expect in expected:
            assert expect == next(fibo_gen)


# Put code here

# Import required libraries

from tools import imputer, impute_scale, impute_one_hot_encode, yes_no_to_binary
from models import models
from sklearn.preprocessing import StandardScaler

import seaborn as sns
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.neighbors import KNeighborsClassifier
from imblearn.over_sampling import SMOTE
from sklearn.decomposition import PCA
from sklearn.model_selection import cross_val_score

from tools import plot_confusion_matrix

import pandas as pd
from sklearn.linear_model import LogisticRegression
from imblearn.ensemble import BalancedBaggingClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.naive_bayes import GaussianNB
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier

from sklearn.metrics import accuracy_score, confusion_matrix, classification_report, f1_score, roc_auc_score

def impute_scale(X_train, X_test):
    # Separate the numerical and categorical features
    numerical_features = X_train.select_dtypes(include=[np.number]).columns.tolist()
    categorical_features = X_train.select_dtypes(include=object).columns.tolist()
    #X_train, X_test = imputer(X_train, X_test)
    scaler = StandardScaler()
    X_train[numerical_features] = scaler.fit_transform(X_train[numerical_features])
    X_test[numerical_features] = scaler.transform(X_test[numerical_features])
    return X_train, X_test

def models(X_train, X_test, y_train, y_test, labels):
    results = []
    print('Building Models...')

    # 5. Decision Tree Classifier
    method_name = 'Decision Tree Classifier'
    print(f'Building {method_name} model...')
    classifier = DecisionTreeClassifier(criterion='entropy', random_state=0)
    classifier.fit(X_train, y_train)
    y_pred = classifier.predict(X_test)
    y_pred_proba = classifier.predict_proba(X_test)[:, 1]
    accuracy = accuracy_score(y_test, y_pred)
    roc = roc_auc_score(y_test, y_pred_proba)
    # print("Decision Tree Accuracy: ", dtt_accuracy)
    # print("Decision Tree ROC AUC: ", dtt_roc)
    cm = confusion_matrix(y_test, y_pred)
    plot_confusion_matrix(cm, labels, method_name)
    report = classification_report(y_test, y_pred, output_dict=True)
    precision = report['weighted avg']['precision']
    recall = report['weighted avg']['recall']
    f1_score = report['weighted avg']['f1-score']
    results.append([method_name, accuracy, roc, precision, recall, f1_score])

    # 6. Random Forest Classifier
    method_name = 'Random Forest Classifier'
    print(f'Building {method_name} model...')
    classifier = RandomForestClassifier(n_estimators=10, criterion='entropy', random_state=0)
    classifier.fit(X_train, y_train)
    y_pred = classifier.predict(X_test)
    y_pred_proba = classifier.predict_proba(X_test)[:, 1]
    accuracy = accuracy_score(y_test, y_pred)
    roc = roc_auc_score(y_test, y_pred_proba)
    # print("Random Forest Accuracy: ", rf_accuracy)
    # print("Random Forest ROC AUC: ", rf_roc)
    cm = confusion_matrix(y_test, y_pred)
    plot_confusion_matrix(cm, labels, method_name)
    report = classification_report(y_test, y_pred, output_dict=True)
    precision = report['weighted avg']['precision']
    recall = report['weighted avg']['recall']
    f1_score = report['weighted avg']['f1-score']
    results.append([method_name, accuracy, roc, precision, recall, f1_score])

    # 7. Gaussian
    method_name = 'Naive Bayes'
    print(f'Building {method_name} model...')
    classifier = GaussianNB()
    classifier.fit(X_train, y_train)
    y_pred = classifier.predict(X_test)
    y_pred_proba = classifier.predict_proba(X_test)[:, 1]
    accuracy = accuracy_score(y_test, y_pred)
    roc = roc_auc_score(y_test, y_pred_proba)
    # print("Naive Bayes Accuracy: ", nb_accuracy)
    # print("Naive Bayes ROC AUC: ", nb_roc)
    cm = confusion_matrix(y_test, y_pred)
    plot_confusion_matrix(cm, labels, method_name)
    report = classification_report(y_test, y_pred, output_dict=True)
    precision = report['weighted avg']['precision']
    recall = report['weighted avg']['recall']
    f1_score = report['weighted avg']['f1-score']
    results.append([method_name, accuracy, roc, precision, recall, f1_score])

    results_df = pd.DataFrame(results, columns=['Model', 'Accuracy', 'ROC AUC', 'Precision', 'Recall', 'F1-score'])
    print(results_df)

df = pd.read_csv('weatherAUS.csv')
df = df.dropna(subset=['RainTomorrow'])
labels = ['No', 'Yes']
print(df.head(10))
print(df.info())
print(df.isnull().sum())

df['RainTomorrow'] = df['RainTomorrow'].apply(yes_no_to_binary)

df['Date'] = pd.to_datetime(df['Date'])
df['Year'] = df['Date'].dt.year
df['Month'] = df['Date'].dt.month
df['Day'] = df['Date'].dt.day
df = df.drop('Date', axis=1)

#df = df.dropna(axis=0, how='any', subset=["RainTomorrow"])

print(df.shape)

X = df.drop('RainTomorrow', axis=1)
y = df['RainTomorrow']

# Perform preprocessing for numerical features
X = imputer(X)

# 6 columns are of type 'object' and remaining of 'float'
cat, cont = [], []

# # MODELLING CODE STARTS HERE

# Split the dataset into training and testing sets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

print(X_train.info())
print(X_train.info())

#X_train, X_test = impute_scale(X_train, X_test)
X_train, X_test = impute_one_hot_encode(X_train, X_test)

# models(X_train, X_test, y_train, y_test, labels)
sns.countplot(x = y_train, hue = df['RainTomorrow'])
plt.show()

sm = SMOTE()
#(sampling_strategy='minority', random_state=42)

# Fit and resample the data
X_train, y_train = sm.fit_resample(X_train, y_train)
print(y_train)

sns.countplot(x = y_train)
plt.show()
models(X_train, X_test, y_train, y_test, labels)

# from lazypredict.Supervised import LazyClassifier
# clf = LazyClassifier(verbose=0, ignore_warnings=True, custom_metric=None)
# models, predictions = clf.fit(X_train, X_test, y_train, y_test)
#
# print(models)

pca = PCA()
pca.fit(X_train)
cumsum = np.cumsum(pca.explained_variance_ratio_)*100
d = [n for n in range(len(cumsum))]
plt.figure(figsize=(10, 10))
plt.plot(d, cumsum, color='red', label='cumulative explained variance')
plt.title('Cumulative Explained Variance as a Function of the Number of Components')
plt.ylabel('Cumulative Explained variance')
plt.xlabel('Principal components')
plt.axhline(y=95, color='k', linestyle='--', label='95% Explained Variance')
plt.legend(loc='best')

pca_train = PCA(.95)
pca_train.fit(X_train)
X_train_pca = pca_train.transform(X_train)
# pca_test = PCA(.95)
# pca_test.fit(X_test)
X_test_pca = pca_train.transform(X_test)

print('After PCA:')
models(X_train_pca, X_test_pca, y_train, y_test, labels)

print('k-Fold Cross Validation:')
# k = n_neighbors
knn = KNeighborsClassifier(n_neighbors=3)
# K Fold CV K=10
accuracies = cross_val_score(estimator=knn, X=X_train, y=y_train, cv=10)
print(accuracies)

print("average accuracy: ", np.mean(accuracies))
print("average std: ", np.std(accuracies))

knn.fit(X_train, y_train)
# print("test accuracy: ", knn.score(X_test,y_test))


import sys
import csv
import numpy as np
import torch

import matplotlib as mpl
import matplotlib.pyplot as plt
import seaborn as sns

import readwrite as rw
from bilinear import WordVectors, MarginalLogLinear, ConditionalSoftmax, ConditionalLogBilinear

def filter_matrix(matrix, labels, subset_labels):
	indices = [labels.index(x) for x in subset_labels]
	matrix = matrix.take(indices, axis=0).take(indices, axis=1)
	# breakpoint()
	return matrix

def heatmap(model_filename, tie):

	model = torch.load(model_filename)

	A_matrix = model.bilinear.weight[0].detach().numpy()
	# plt.imshow(A_matrix, cmap='hot', interpolation='nearest')
	# plt.show()

	full_labels = ['syll','cons','son','cont','del rel','appr','nas','voi','sp glo','labi','rd','labiodent','cor','ant','dist','str','later','dors','high','low','front','back','tense']
	labels = ['cons','son','cont','del rel','appr']

	if not int(tie):
		full_labels = [y for x in full_labels for y in ('+' + x, '-' + x)]

		labels = [y for x in labels for y in ('+' + x, '-' + x)]

	# tied labels mean the weights for negative features and postive features are the same

	# s1 = TH-_ W-_ S-_ T-_ V-_ HH-_ D-_ R-_ N-_ K-_ F-_ CH-_ B-_ L-_ DH-_ JH-_ G-_ P-_ Z-_ M-_ Y-_ SH-_

	# s2 =_-TH _-W _-# _-S _-T _-V _-HH _-D _-R _-N _-K _-F _-CH _-B _-L _-DH _-JH _-G _-P _-Z _-M _-Y _-SH #-_

	A_matrix = filter_matrix(A_matrix, full_labels, labels)
	print(A_matrix)

	fig, ax = plt.subplots()
	im = ax.imshow(A_matrix, cmap="seismic")

	# Show l ticks and label them with the respective list entries
	ax.set_xticks(np.arange(len(labels)), labels=labels)
	ax.set_yticks(np.arange(len(labels)), labels=labels)

	# Rotate the tick labels and set their ignment.
	plt.setp(ax.get_xticklabels(), rotation=90, ha="right",
			rotation_mode="anchor")

	ax.set_title("A_matrix")
	fig.tight_layout()
	# plt.show()

	print(model.w_linear.weight)
	plt.savefig('heatmap_%s.png' %str(model_filename).split(".")[0].replace("result/",""), dpi=300)

if __name__ == '__main__':
	heatmap(*sys.argv[1:])


from flask import Flask
from flask_session import Session
from flask_caching import Cache

from app.models import db
from config import Config
from app.routes import register_routes

cache = Cache()

def create_app():
    app = Flask(__name__, instance_relative_config=True)
    app.config.from_object(Config.get_config())

    cache.init_app(app)
    db.init_app(app)
    register_routes(app)
    Session(app)

    return app

if __name__ == '__main__':
    app = create_app()
    app.run()
else:
    gunicorn_app = create_app()


from dataclasses import dataclass
from typing import List, Set, Tuple
from math import inf
from argparse import ArgumentParser
import time
from itertools import combinations

@dataclass
class Valve:
    flowr: int
    direct: list
    distance: dict

def bad_bfs(valves: dict, starter: str):
    to_check = [starter]

    distances = dict()
    for k in valves.keys():
        if k == starter:
            distances.update({k: 0})
        else:
            distances.update({k: inf})

    while to_check:
        current = to_check.pop(0)
        for nbr in valves[current].direct:
            if distances[nbr] == inf:
                to_check.append(nbr)
            if distances[nbr] > distances[current] + 1:
                distances[nbr] = distances[current] + 1

    valves[starter].distance = distances

def parse_input(lines):
    valves = dict()
    # some dirty parsing to get all valves
    for line in lines:
        name = line[6:8]
        flowrate = int(line[line.index('=') + 1: line.index(';')])
        if 'valves' in line:
            direct = [neighbour.strip() for neighbour in line[line.index('valves') + 7:].split(', ')]
        else:
            direct = [line[line.index('valve') + 6: line.index('valve') + 8]]
        valves.update({name: Valve(flowrate, direct, dict())})
    return valves

if __name__ == "__main__":
    parser = ArgumentParser()
    parser.add_argument('-i', '--input', type=str, default='test.txt')
    parser.add_argument('-c', '--challenge', type=int, default=1)
    args = parser.parse_args()

    with open(args.input, 'r') as inp:
        lines = [line.strip() for line in inp.readlines()]

        '''
        IDEA:
            - get distances for each valve to all other valves
            - simplify graph by removing all 0 flow-rate nodes
            - brute force it
        '''
        tp0 = time.time_ns()
        valves = parse_input(lines)
        tp = (time.time_ns() - tp0) / 10**9
        print(f"Time parsing: {tp: .5f}s")

        # get all distances
        td0 = time.time_ns()
        for vname in valves.keys():
            bad_bfs(valves, vname)
        td = (time.time_ns() - td0) / 10**9
        print(f"Time for distances: {td: .5f}s")

        # remove all 0 flows except AA
        print(f"Total valves = {len(valves)}")
        discard = list()
        for vname in valves.keys():
            if vname != 'AA' and valves[vname].flowr == 0:
                discard.append(vname)
        print(f"Discarding {len(discard)} valves with 0 flow")
        for d in discard:
            valves.pop(d)
        for vname in valves.keys():
            for d in discard:
                valves[vname].distance.pop(d)
        relevant_valve_count = len(valves)
        print(f"Remaining valves {relevant_valve_count}")

        if args.challenge == 1:
            # brute force challenge 1
            def max_release(position: str, remaining_minutes: int, already_open: List[str]) -> int:
                if remaining_minutes <= 0:
                    return 0
                release_from_this = remaining_minutes * valves[position].flowr
                options = list(filter(lambda v: v not in already_open and v != position and v != 'AA', valves.keys()))
                if not options:
                    return release_from_this

                possibilities = [max_release(next, remaining_minutes - valves[position].distance[next] - 1, already_open.copy() + [position]) for next in options]
                return max(possibilities) + release_from_this

            ts0 = time.time_ns()
            solution = max_release('AA', 30, [])
            ts = (time.time_ns() - ts0) / 10**9
            print(f"Time to solve: {ts: .5f}s")
        else:
            # brute force with the elephant helper
            def max_release_el(position_me: str, position_el: str, worktime_me: int, worktime_el: int, remaining_minutes: int, already_targeted: List[str], h) -> Tuple[int, List]:
                if remaining_minutes <= 0:
                    return 0, []

                # one of the worktimes has to be 0 always (either I or elephant opened valve)
                if worktime_me != 0 and worktime_el != 0:
                    raise Exception("You implemented some dumb stuff. worktimes both nonzero")

                # am I or the elephant opening a valve this minute
                release_me = remaining_minutes * valves[position_me].flowr if worktime_me == 0 else 0
                release_el = remaining_minutes * valves[position_el].flowr if worktime_el == 0 else 0
                release_from_this = release_el + release_me

                options = list(filter(lambda v: v not in already_targeted and v != position_me and v != position_el and v != 'AA', valves.keys()))

                if release_me > 0:
                    h.append(f"I open valve {position_me} with {remaining_minutes} minutes remaining for {release_me} pressure. Options: {options}")
                if release_el > 0:
                    h.append(f"The elephant opens valve {position_el} with {remaining_minutes} minutes remaining for {release_el} pressure. Options: {options}")

                # without any decisions remaining calculate release for this configuration
                if not options:
                    if worktime_me > 0 and remaining_minutes - worktime_me > 0:
                        future_release_me = (remaining_minutes - worktime_me) * valves[position_me].flowr
                        h.append(f"Lastly I open valve {position_me} with {remaining_minutes - worktime_me} minutes remaining for {future_release_me} pressure.")
                    else:
                        future_release_me = 0
                    if worktime_el > 0 and remaining_minutes - worktime_el > 0:
                        future_release_el = (remaining_minutes - worktime_el) * valves[position_el].flowr
                        h.append(f"Lastly the elephant opens valve {position_el} with {remaining_minutes - worktime_el} minutes remaining for {future_release_el} pressure.")
                    else:
                        future_release_el = 0
                    return release_from_this + future_release_me + future_release_el, h

                # check who needs a new target valve (position)
                if worktime_me == 0 and worktime_el == 0:
                    # new target for both
                    if len(options) == 1:
                        # can only open one more valve
                        last_target = options.pop(0)
                        min_dist = min(valves[position_me].distance[last_target], valves[position_el].distance[last_target])
                        from_last = max((remaining_minutes - (min_dist + 1)) * valves[last_target].flowr, 0)
                        h.append(f"One of us opens valve {last_target} with {remaining_minutes - min_dist - 1} minutes remaining.")
                        return release_from_this + from_last, h

                    target_combinations = list(combinations(options, 2))
                    best_case, h_best = -1, []
                    for t_el, t_me in target_combinations:
                        new_worktime_me = valves[position_me].distance[t_me] + 1
                        new_worktime_el = valves[position_el].distance[t_el] + 1
                        next_choice_in = min(new_worktime_me, new_worktime_el)

                        comb_solution, h_new = max_release_el(t_me, t_el,
                                                       new_worktime_me - next_choice_in,
                                                       new_worktime_el - next_choice_in,
                                                       remaining_minutes - next_choice_in,
                                                       already_targeted.copy() + [t_me, t_el],
                                                       h.copy())

                        if comb_solution >= best_case:
                            best_case, h_best = comb_solution, h_new
                    return best_case + release_from_this, h_best

                elif worktime_me == 0 and worktime_el != 0:
                    # new target valve for me
                    best_case, h_best = -1, []
                    for t_me in options:
                        new_worktime_me = valves[position_me].distance[t_me] + 1
                        next_choice_in = min(new_worktime_me, worktime_el)
                        target_solution, h_new = max_release_el(t_me, position_el,
                                                         new_worktime_me - next_choice_in,
                                                         worktime_el - next_choice_in,
                                                         remaining_minutes - next_choice_in,
                                                         already_targeted.copy() + [t_me],
                                                         h.copy())
                        if target_solution >= best_case:
                            best_case, h_best = target_solution, h_new
                    return best_case + release_from_this, h_best
                elif worktime_me != 0 and worktime_el == 0:
                    # new target valve for el
                    best_case, h_best = -1, []
                    for t_el in options:
                        new_worktime_el = valves[position_el].distance[t_el] + 1
                        next_choice_in = min(worktime_me, new_worktime_el)
                        target_solution, h_new = max_release_el(position_me, t_el,
                                                         worktime_me - next_choice_in,
                                                         new_worktime_el - next_choice_in,
                                                         remaining_minutes - next_choice_in,
                                                         already_targeted.copy() + [t_el],
                                                         h.copy())
                        if target_solution >= best_case:
                            best_case, h_best = target_solution, h_new
                    return best_case + release_from_this, h_best
                else:
                    raise Exception("Worktime Error")

            ts0 = time.time_ns()
            solution, hist = max_release_el('AA', 'AA', 0, 0, 26, [], [])
            ts = (time.time_ns() - ts0) / 10 ** 9
            print(f"Time to solve: {ts: .5f}s")
            for moment in hist:
                print(moment)

        print(f"Challenge {args.challenge}: {solution}")


import boto3
import pandas as pd
import numpy as np
import pickle
import os
import requests
import json
import zipfile
import logging
import time
from time import strftime, gmtime, sleep
from botocore.auth import SigV4Auth
from botocore.awsrequest import AWSRequest
from datetime import datetime
from urllib.parse import urlparse
from sagemaker.s3 import S3Downloader

# How often to check the status
UPDATE_DELAY_SECONDS = 15
HOME_DIRECTORY = os.path.expanduser("~")

def signed_request(method, url, data=None, params=None, headers=None, service=None):
    request = AWSRequest(
        method=method, url=url, data=data, params=params, headers=headers
    )
    session = boto3.Session()
    credentials = session.get_credentials()
    try:
        frozen_creds = credentials.get_frozen_credentials()
    except AttributeError:
        print("Could not find valid IAM credentials in any the following locations:\n")
        print(
            "env, assume-role, assume-role-with-web-identity, sso, shared-credential-file, custom-process, "
            "config-file, ec2-credentials-file, boto-config, container-role, iam-role\n"
        )
        print(
            "Go to https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html for more "
            "details on configuring your IAM credentials."
        )
        return request
    SigV4Auth(frozen_creds, service, boto3.Session().region_name).add_auth(request)
    return requests.request(
        method=method, url=url, headers=dict(request.headers), data=data
    )

def load_configuration():
    with open(f"{HOME_DIRECTORY}/graph_notebook_config.json") as f:
        data = json.load(f)
        host = data["host"]
        port = data["port"]
        if data["auth_mode"] == "IAM":
            iam = True
        else:
            iam = False
    return host, port, iam

def get_host():
    host, port, iam = load_configuration()
    return host

def get_training_job_name(prefix: str):
    return f"{prefix}-{int(time.time())}"

def check_ml_enabled():
    host, port, use_iam = load_configuration()
    response = signed_request(
        "GET", url=f"https://{host}:{port}/ml/modeltraining", service="neptune-db"
    )
    if response.status_code != 200:
        print(
            """This Neptune cluster \033[1mis not\033[0m configured to use Neptune ML.
Please configure the cluster according to the Amazpnm Neptune ML documentation before proceeding."""
        )
    else:
        print("This Neptune cluster is configured to use Neptune ML")

def get_neptune_ml_job_output_location(job_name: str, job_type: str):
    assert job_type in [
        "dataprocessing",
        "modeltraining",
        "modeltransform",
    ], "Invalid neptune ml job type"

    host, port, use_iam = load_configuration()

    response = signed_request(
        "GET",
        service="neptune-db",
        url=f"https://{host}:{port}/ml/{job_type}/{job_name}",
        headers={"content-type": "application/json"},
    )
    result = json.loads(response.content.decode("utf-8"))
    if result["status"] != "Completed":
        logging.error(
            "Neptune ML {} job: {} is not completed".format(job_type, job_name)
        )
        return
    return result["processingJob"]["outputLocation"]

def get_modeltraining_job_output_location(training_job_name: str):
    assert (
        training_job_name is not None
    ), "Neptune ML training job name id should be passed, if training job s3 output is missing"
    return get_neptune_ml_job_output_location(training_job_name, "modeltraining")

def get_embeddings(
    training_job_name: str, download_location: str = "./model-artifacts"
):
    training_job_s3_output = get_modeltraining_job_output_location(training_job_name)
    if not training_job_s3_output:
        return

    download_location = os.path.join(download_location, training_job_name)
    os.makedirs(download_location, exist_ok=True)
    # download embeddings and mapping info

    S3Downloader.download(
        os.path.join(training_job_s3_output, "embeddings/"),
        os.path.join(download_location, "embeddings/"),
    )

    entity_emb = np.load(os.path.join(download_location, "embeddings", "entity.npy"))

    return entity_emb

def get_mapping(training_job_name: str, download_location: str = "./model-artifacts"):
    training_job_s3_output = get_modeltraining_job_output_location(training_job_name)
    if not training_job_s3_output:
        return

    download_location = os.path.join(download_location, training_job_name)
    os.makedirs(download_location, exist_ok=True)
    # download embeddings and mapping info

    S3Downloader.download(training_job_s3_output, download_location)


import torch
from math import sqrt
from torch import nn

D_MODEL = 512

def manual_SDPA(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> torch.Tensor:
    """
    Args:
        q (tensor): The queries, of dimension (*, d_k, D_MODEL).
        k (tensor): The keys, of dimension (*, d_k, D_MODEL).
        v (tensor): The values, of dimension (*, d_v, D_MODEL).
    """
    q_mul_k = torch.matmul(q, k.transpose(-2, -1))
    scaled = torch.div(q_mul_k, sqrt(k.shape[-1]))
    e_powered = torch.exp(scaled)
    softmaxed = torch.div(e_powered, torch.sum(e_powered).unsqueeze(-1))
    return torch.matmul(softmaxed, v)

class manual_MHA(nn.Module):
    def __init__(self, num_heads: int):
        """
        Args:
            num_heads (int): Number of heads to concatenate together at end.
        """
        super().__init__()
        self.num_heads = num_heads
        assert D_MODEL % num_heads == 0, \
            f"D_MODEL / num_heads must divide evenly, not {D_MODEL} / {num_heads}!"
        self.d_k = int(D_MODEL / num_heads)
        self.d_v = int(D_MODEL / num_heads)
        self.q_lin_projs = nn.Parameter(torch.randn(num_heads,
                                                    D_MODEL,
                                                    self.d_k))
        self.k_lin_projs = nn.Parameter(torch.randn(num_heads,
                                                    D_MODEL,
                                                    self.d_k))
        self.v_lin_projs = nn.Parameter(torch.randn(num_heads,
                                                    D_MODEL,
                                                    self.d_v))
        self.concat_lin_proj = nn.Parameter(torch.randn(num_heads * self.d_v,
                                                        D_MODEL))

    def forward(self, embeded_seq: torch.Tensor) -> torch.Tensor:
        """
        Args:
            embeded_seq (tensor): A matrix of the embeded sequence, of
                dimension (seq_len, D_MODEL).
        """
        # Create matrix with a copy of the input sequence for each head
        expanded_seq = embeded_seq.expand(self.num_heads, -1, -1)

        # Results have dimensions of (num_heads, seq_len, d_k or d_v)
        qs = torch.matmul(expanded_seq, self.q_lin_projs)
        ks = torch.matmul(expanded_seq, self.k_lin_projs)
        vs = torch.matmul(expanded_seq, self.v_lin_projs)

        print(f"qs: {qs.shape}")
        print(f"ks: {ks.shape}")
        print(f"vs: {vs.shape}")

        # Calculate attentions
        head_attns = manual_SDPA(qs, ks, vs)

        print(f"head_attns: {head_attns.shape}")

        # Concat head outputs to dimensions of (seq_len, self.num_heads * self.d_v)
        concat_heads = torch.cat([head_attn.squeeze(dim=0) for head_attn in torch.split(head_attns, 1, dim=0)], dim=1)

        return torch.matmul(concat_heads, self.concat_lin_proj)

class manual_encoder(nn.Module):
    def __init__(self, d_model: int):
        """
        Args:
            d_model (int): Dimension of embeddings and tensors passed between
                internal blocks of the encoder and decoder.
        """
        super().__init__()

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        pass

class manual_decoder(nn.Module):
    def __init__(self, ):
        """
        Args:

        """

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        pass

if __name__ == "__main__":
    sdpa = manual_SDPA()


import argparse
args = argparse.ArgumentParser()
args.add_argument("--config_path", help="Path to Config File", default="configs/train_habitat.yaml")
args = args.parse_args()

from configs import get_cfg_defaults

from trainer import Trainer

if __name__ == "__main__":

    cfg = get_cfg_defaults()

    if args.config_path[-4:] == 'yaml':
        cfg.merge_from_file(args.config_path)
    else:
        print("No valid config specified, using default")

    cfg.freeze()
    print(cfg)

    # Train stage 1
    # Assume that it has been trained, and it SAVES all the depth outputs in a

    train = Trainer(cfg)
    train.train()


"""Non-CRU API and composite messages"""

import logging
import webapp2

from google.appengine.ext import ndb
from protorpc import message_types
from protorpc import messages
from protorpc import remote
from protorpc.wsgi import service

import base_api
import ndb_models
import protorpc_messages

package = 'rooms'

class OrderFormChoice(messages.Message):
  id = messages.IntegerField(1)
  name = messages.StringField(2)
  code = messages.StringField(3)
  visibility = messages.StringField(4)

class OrderFormChoices(messages.Message):
  order_form = messages.MessageField(OrderFormChoice, 1, repeated=True)

class OrderFormDetail(messages.Message):
  order_sheet = messages.MessageField(protorpc_messages.OrderSheet, 1)
  sorted_items = messages.MessageField(protorpc_messages.Item, 2, repeated=True)

class OrderFull(messages.Message):
  order = messages.MessageField(protorpc_messages.Order, 1, required=True)
  order_items = messages.MessageField(protorpc_messages.OrderItem, 2, repeated=True)
  delivery = messages.MessageField(protorpc_messages.Delivery, 3)
  pickup = messages.MessageField(protorpc_messages.Pickup, 4)
  borrow = messages.MessageField(protorpc_messages.Borrow, 7)
  retrieval = messages.MessageField(protorpc_messages.Retrieval, 5)
  id = messages.IntegerField(6)

class OrderCheckExisting(messages.Message):
  ordersheet_id = messages.IntegerField(1)
  site_id = messages.IntegerField(2)

class OrderExisting(messages.Message):
  order = messages.MessageField(protorpc_messages.Order, 1, repeated=True)

class LogisticsDate(messages.Message):
  date = messages.StringField(1)
  order_id = messages.IntegerField(2)
  logistics_type = messages.StringField(3)
  ordersheet_name = messages.StringField(4)

class LogisticsDates(messages.Message):
  logistics_date = messages.MessageField(LogisticsDate, 1, repeated=True)

class SiteCaptainDetail(messages.Message):
  sitecaptain = messages.MessageField(protorpc_messages.SiteCaptain, 1)
  name = messages.StringField(2)

class SiteCaptains(messages.Message):
  sitecaptain_detail = messages.MessageField(SiteCaptainDetail, 1, repeated=True)

class CustomApi(base_api.BaseApi):
  """Protorpc service implementing a custom API for ROOM.

  This covers multi-model updates and more complex reads.
  """

  @remote.method(message_types.VoidMessage,
                 message_types.VoidMessage)
  def ehlo(self, request):
    self._authorize_user()
    logging.info('ehlo')
    return message_types.VoidMessage()

  @remote.method(message_types.VoidMessage,
                 OrderFormChoices)
  def order_form_choices(self, request):
    self._authorize_user()
    res = OrderFormChoices()
    for m in ndb_models.OrderSheet.query(ndb_models.OrderSheet.visibility != 'Inactive'):
      f = OrderFormChoice(
        id=m.key.integer_id(), name=m.name,
        code=m.code, visibility=m.visibility)
      res.order_form.append(f)
    return res

  @remote.method(OrderCheckExisting,
                 OrderExisting)
  def order_existing(self, request):
    self._authorize_user()
    res = OrderExisting()
    site_key = ndb.Key(ndb_models.NewSite, request.site_id)
    ordersheet_key = ndb.Key(ndb_models.OrderSheet, request.ordersheet_id)

    orders = sorted(ndb_models.Order.query(
        ndb_models.Order.state!='Deleted',
        ndb_models.Order.order_sheet==ordersheet_key,
        ndb_models.Order.site==site_key), key=lambda o: o.modified, reverse=True)
    for mdl in orders:
      msg = protorpc_messages.OrderModelToMessage(mdl)
      res.order.append(msg)
    return res

  @remote.method(protorpc_messages.SimpleId,
                 LogisticsDates)
  def logistics_dates(self, request):
    self._authorize_user()
    res = LogisticsDates()
    site_key = ndb.Key(ndb_models.NewSite, request.id)
    orders = ndb_models.Order.query(ndb_models.Order.state!='Deleted',
                                    ndb_models.Order.site==site_key)
    existing_dates = []
    for order in orders:
      os = order.order_sheet.get()

      for od in ndb_models.OrderDelivery.query(ndb_models.OrderDelivery.order==order.key):
        d = od.delivery.get()
        if d.delivery_date:
          existing_dates.append(
            LogisticsDate(date=d.delivery_date, logistics_type='Delivery',
                          ordersheet_name=os.name, order_id=order.key.integer_id()))

      for op in ndb_models.OrderPickup.query(ndb_models.OrderPickup.order==order.key):
        p = op.pickup.get()
        if p.pickup_date:
          existing_dates.append(
            LogisticsDate(date=p.pickup_date, logistics_type='Pick-up',
                          ordersheet_name=os.name, order_id=order.key.integer_id()))
        if p.return_date:
          existing_dates.append(
            LogisticsDate(date=p.return_date, logistics_type='Return',
                          ordersheet_name=os.name, order_id=order.key.integer_id()))

      for ob in ndb_models.OrderBorrow.query(ndb_models.OrderBorrow.order==order.key):
        p = ob.borrow.get()
        if p.borrow_date:
          existing_dates.append(
            LogisticsDate(date=p.borrow_date, logistics_type='Borrow',
                          ordersheet_name=os.name, order_id=order.key.integer_id()))
        if p.return_date:
          existing_dates.append(
            LogisticsDate(date=p.return_date, logistics_type='Return',
                          ordersheet_name=os.name, order_id=order.key.integer_id()))

      for orl in ndb_models.OrderRetrieval.query(ndb_models.OrderRetrieval.order==order.key):
        r = orl.retrieval.get()
        if r.dropoff_date:
          existing_dates.append(
            LogisticsDate(date=r.dropoff_date, logistics_type='Drop-off',
                          ordersheet_name=os.name, order_id=order.key.integer_id()))
        if r.retrieval_date:
          existing_dates.append(
            LogisticsDate(date=r.retrieval_date, logistics_type='Retrieval',
                          ordersheet_name=os.name, order_id=order.key.integer_id()))

    res.logistics_date = sorted(existing_dates, key=lambda e: e.date)
    return res

  @remote.method(protorpc_messages.SimpleId, OrderFormDetail)
  def order_form_detail(self, request):
    self._authorize_user()
    res = OrderFormDetail()
    key = ndb.Key(ndb_models.OrderSheet, request.id)
    mdl = key.get()
    if not mdl:
      raise remote.ApplicationError(
        'No OrderSheet found with key {}'.format(request.id))
    res.order_sheet = protorpc_messages.OrderSheetModelToMessage(mdl)
    ims = list(ndb_models.Item.query(ndb_models.Item.appears_on_order_form == key))
    ndb_models._SortItemsWithSections(ims)
    for im in ims:
      i = protorpc_messages.ItemModelToMessage(im)
      res.sorted_items.append(i)
    return res

  @remote.method(protorpc_messages.SimpleId, OrderFull)
  def order_full_read(self, request):
    self._authorize_user()
    res = OrderFull()
    order_key = ndb.Key(ndb_models.Order, request.id)
    order_mdl = order_key.get()
    if order_mdl is None:
      raise remote.ApplicationError(
        'No Order found with key {}'.format(request.id))
    res.id = order_key.integer_id()
    res.order = protorpc_messages.OrderModelToMessage(order_mdl)

    for oi_mdl in ndb_models.OrderItem.query(ndb_models.OrderItem.order == order_key):
      res.order_items.append(protorpc_messages.OrderItemModelToMessage(oi_mdl))

    join_mdl = ndb_models.OrderDelivery.query(ndb_models.OrderDelivery.order == order_key).get()
    if join_mdl is not None:
      res.delivery = protorpc_messages.DeliveryModelToMessage(join_mdl.delivery.get())

    join_mdl = ndb_models.OrderPickup.query(ndb_models.OrderPickup.order == order_key).get()
    if join_mdl is not None:
      res.pickup = protorpc_messages.PickupModelToMessage(join_mdl.pickup.get())

    join_mdl = ndb_models.OrderBorrow.query(ndb_models.OrderBorrow.order == order_key).get()
    if join_mdl is not None:
      res.borrow = protorpc_messages.BorrowModelToMessage(join_mdl.borrow.get())

    join_mdl = ndb_models.OrderRetrieval.query(ndb_models.OrderRetrieval.order == order_key).get()
    if join_mdl is not None:
      res.retrieval = protorpc_messages.RetrievalModelToMessage(join_mdl.retrieval.get())

    return res

  def _order_full_put(self, request, order=None):
    # TODO ndb.start_transaction ...
    if order is None:
      order = ndb_models.Order()
    order = protorpc_messages.OrderMessageToModel(request.order, order)
    sub_total = 0.
    oimdl_has_order_key = []
    oimdl_needs_order_key = []

    if not order.state:
      order.state = 'Received'

    for oimsg in request.order_items:
      # Existing order item, changed to 0 quantity, so delete it.
      if oimsg.id and not oimsg.quantity:
        ndb.Key(ndb_models.OrderItem, oimsg.id).delete()
        continue

      # 0 quantity, we don't need to add it.
      if not oimsg.quantity:
        continue

      # Add/Update the OrderItem.
      if oimsg.id:
        oimdl = ndb.Key(ndb_models.OrderItem, oimsg.id).get()
        oimdl_has_order_key.append(oimdl)
      else:
        if order.key:
          oimdl = ndb_models.OrderItem(order=order.key)
          oimdl_has_order_key.append(oimdl)
        else:
          oimdl = ndb_models.OrderItem(order=ndb.Key(ndb_models.Order, 'placeholder'))
          oimdl_needs_order_key.append(oimdl)

      oimdl = protorpc_messages.OrderItemMessageToModel(
        oimsg, oimdl)

      if order.state == 'Received' or oimdl.unit_cost is None:
        oimdl.unit_cost = oimdl.item.get().unit_cost

      if oimdl.quantity_float and oimdl.unit_cost:
        sub_total += oimdl.quantity_float * oimdl.unit_cost

    order.sub_total = sub_total
    order.put()

    for oimdl in oimdl_needs_order_key:
      oimdl.order = order.key
      oimdl_has_order_key.append(oimdl)

    ndb.put_multi(oimdl_has_order_key)

    if request.delivery:
      if request.delivery.id:
        delivery = ndb.Key(ndb_models.Delivery, request.delivery.id).get()
        if not delivery:
          logging.error('no Delivery found with id {}'.format(request.delivery.id))
          webapp2.abort(404)
      else:
        delivery = ndb_models.Delivery(site=order.site)

      protorpc_messages.DeliveryMessageToModel(request.delivery, delivery)
      delivery.put()
      if not request.delivery.id:
        ndb_models.OrderDelivery(order=order.key, delivery=delivery.key).put()

    if request.pickup:
      if request.pickup.id:
        pickup = ndb.Key(ndb_models.Pickup, request.pickup.id).get()
        if not pickup:
          logging.error('no Pickup found with id {}'.format(request.pickup.id))
          webapp2.abort(404)
      else:
        pickup = ndb_models.Pickup(site=order.site)

      protorpc_messages.PickupMessageToModel(request.pickup, pickup)
      pickup.put()
      if not request.pickup.id:
        ndb_models.OrderPickup(order=order.key, pickup=pickup.key).put()

    if request.borrow:
      if request.borrow.id:
        borrow = ndb.Key(ndb_models.Borrow, request.borrow.id).get()
        if not borrow:
          logging.error('no Borrow found with id {}'.format(request.borrow.id))
          webapp2.abort(404)
      else:
        borrow = ndb_models.Borrow(site=order.site)

      protorpc_messages.BorrowMessageToModel(request.borrow, borrow)
      borrow.put()
      if not request.borrow.id:
        ndb_models.OrderBorrow(order=order.key, borrow=borrow.key).put()

    if request.retrieval:
      if request.retrieval.id:
        retrieval = ndb.Key(ndb_models.Retrieval, request.retrieval.id).get()
        if not retrieval:
          logging.error('no Retrieval found with id {}'.format(request.retrieval.id))
          webapp2.abort(404)
      else:
        retrieval = ndb_models.Retrieval(site=order.site)

      protorpc_messages.RetrievalMessageToModel(request.retrieval, retrieval)
      retrieval.put()
      if not request.retrieval.id:
        ndb_models.OrderRetrieval(order=order.key, retrieval=retrieval.key).put()

    order.UpdateLogistics()
    return order.key.integer_id()

  @remote.method(OrderFull, protorpc_messages.SimpleId)
  def order_full_create(self, request):
    self._authorize_user()
    logging.info("order_full_create(%s)", request)
    if request.id:
      raise remote.ApplicationError('must not have id in create')
    order_id = self._order_full_put(request)
    return protorpc_messages.SimpleId(id=order_id)

  @remote.method(OrderFull, message_types.VoidMessage)
  def order_full_update(self, request):
    self._authorize_user()
    logging.info("order_full_update(%s)", request)
    if not request.id:
      raise remote.ApplicationError('id is required')
    mdl = ndb.Key(ndb_models.Order, request.id).get()
    if not mdl:
      raise remote.ApplicationError(
        'No Order found with key {}'.format(request.id))
    self._order_full_put(request, mdl)
    return message_types.VoidMessage()

  @remote.method(protorpc_messages.SimpleId, message_types.VoidMessage)
  def order_fulfill(self, request):
    self._authorize_staff()
    order = ndb.Key(ndb_models.Order, request.id).get()
    if not order:
      raise remote.ApplicationError('Order not found: %s' % request.id)
    order.state = 'Being Filled'
    return message_types.VoidMessage()

  @remote.method(protorpc_messages.SimpleId, SiteCaptains)
  def sitecaptains_for_site(self, request):
    self._authorize_user()
    res = SiteCaptains()
    sitecaptain_models = list(
      ndb_models.SiteCaptain.query(ndb_models.SiteCaptain.site == ndb.Key(ndb_models.NewSite, request.id)))
    for m in sitecaptain_models:
      f = protorpc_messages.SiteCaptainModelToMessage(m)
      captain_model = ndb.Key(ndb_models.Captain, f.captain).get()  # TODO: get_multi
      detail = SiteCaptainDetail(sitecaptain=f, name=captain_model.name)
      res.sitecaptain_detail.append(detail)
    return res

application = service.service_mapping(CustomApi, r'/custom_api')


from django.contrib import admin
from django.urls import path,include
from . import views

urlpatterns = [
    path('',views.Registration_User),
    path('Home',views.Home,name = 'Home'),
    path('pdf',views.pdf,name = 'pdf'),
    path('download',views.download,name = 'download'),
    path('Apply_Test/',views.Apply_Test,name = 'Apply_Test/'),
    path('Apply_Vaccine/',views.Apply_Vaccine,name = 'Apply_Vaccine/'),
    path('user_status/',views.user_status,name = 'user_status/'),
]


import streamlit as st
import pandas as pd

import matplotlib.pyplot as plt
import seaborn as sns

import accountanalysis as acc

# ------------------ settings --------------------
st.set_page_config(layout="wide")

# ---------------------- functions for caching --------------------------

@st.cache
def load_data(uploaded_file, col=0):
    df = pd.read_csv(uploaded_file, index_col=0, parse_dates=["Date"])
    return df

def get_account_information(df, account_name):
    n_send = len(df[df["From"] == account_name].drop_duplicates("Date"))
    n_recieved = len(df[df["To"] == account_name].drop_duplicates("Date"))
    date_first_mail = df[df["From"] == account_name]["Date"].min().date()
    date_last_mail = df[df["From"] == account_name]["Date"].max().date()
    df_poi = acc.filter_df_for_poi(df=df, poi=account_name)
    n_contacts = len(acc.get_contacts_of_poi(df=df_poi, poi=account_name))

    st.write("Anzahl gesendeter E-Mails:", n_send)
    st.write("Anzahl empfangener E-Mails:", n_recieved)
    st.write("Erste gesendete E-Mail:", date_first_mail)
    st.write("Letzte gesendete E-Mail:", date_last_mail)
    st.write("Anzahl Kontakte:", n_contacts)

@st.cache
def create_timeline(df, col_name):
    timeline = acc.create_timeline(df=df, col_name="Date")
    return timeline

@st.cache
def create_communication_timeline(df, poi, timeline):
    df_poi_sent = df[df["From"] == poi][~df[df["From"] == poi].index.duplicated(keep="first")]
    df_poi_recieved = df[df["To"] == poi][~df[df["To"] == poi].index.duplicated(keep="first")]

    com_time_series = pd.DataFrame(index=timeline)

    com_time_series["sent"] = acc.get_cumulated_elements_per_day(df_raw=df_poi_sent, timeline=timeline)
    com_time_series["recieved"] = acc.get_cumulated_elements_per_day(df_raw=df_poi_recieved, timeline=timeline)
    return com_time_series

@st.cache
def get_df_poi(df, poi):
    df_poi = acc.filter_df_for_poi(df=df, poi=poi).drop_duplicates("Date")
    return df_poi

@st.cache
def get_contacts_of_poi(df_poi, poi):
    contacts_of_poi = acc.get_contacts_of_poi(df=df_poi, poi=poi)
    contacts_with_count = acc.get_contacts_with_count(df_poi=df_poi, poi=poi, contacts_of_poi=contacts_of_poi)
    return contacts_with_count.sort_values(by="sent", ascending=False)

session_state_keys = ["file_cached", "is_analyzed", "plot_settings", "is_analyze"]

for key in session_state_keys:
    if key not in st.session_state.keys():
        st.session_state[key] = False

with st.container():
    st.header("Analyse der Kommuniaktion eines E-Mail Accounts")

    # ------------------ load mails --------------------
    st.write("Füge eine .csv Datei mit Texten ein. Die Spalten 'Date', 'From' und 'To' werden erwartet.")

    uploaded_file = st.file_uploader("Choose a file")
    if uploaded_file is not None:
        df = load_data(uploaded_file)
        st.write("Anzahl der Mails:", len(df))
        st.session_state["file_cached"] = True
    else:
        df = None

    # ------------------ settings  --------------------
    if st.session_state["file_cached"]:
        mail_accounts = set(df["From"]).union(set(df["To"]))
        poi = st.sidebar.selectbox("Wähle einen E-Mail Account:", mail_accounts)
        df_poi = get_df_poi(df=df, poi=poi)
        # TODO: filter time

        if st.button("Show E-Mail Activities"):
            st.session_state["is_analyze"] = True

    if st.session_state["is_analyze"]:

        # show general information about the data
        st.subheader("Algemeine Informationen über den Account:")
        get_account_information(df=df, account_name=poi)

        timeline = create_timeline(df=df, col_name="Date")

        # create cumulated mail count of poi
        st.subheader("Kumulierte empfange und gesendete E-Mails")
        com_time_series = create_communication_timeline(df=df, poi=poi, timeline=timeline)

        fig_com_time_series, ax_com_time_series = plt.subplots()

        fig_com_time_series.set_figwidth(20)
        fig_com_time_series.set_figheight(10)

        sns.lineplot(data=com_time_series, x=com_time_series.index, y="sent", label="sent", ax=ax_com_time_series)
        sns.lineplot(data=com_time_series, x=com_time_series.index, y="recieved", label="recieved", ax=ax_com_time_series)
        st.pyplot(fig_com_time_series)

        # show mail distribution during the day of poi
        st.subheader("Verteilung gesendeter und empfangener E-Mails nach Uhrzeit")
        df_time = acc.get_daily_distribution(df_poi=df_poi, poi=poi)

        fig_time_distribution, ax_time_distribution = plt.subplots()

        fig_time_distribution.set_figwidth(20)
        fig_time_distribution.set_figheight(10)

        sns.histplot(data=df_time, x="time", bins=24, hue="action", ax=ax_time_distribution)
        st.pyplot(fig_time_distribution)

        # contacts of poi
        st.subheader("Häufigste Kontakte des E-Mail Accounts")
        contacts_with_count = get_contacts_of_poi(df_poi=df_poi, poi=poi)
        st.write(contacts_with_count.sort_values("sent", ascending=False))

        kind_of_mail = st.text_input("Gesendete oder empfangene E-Mails betrachten?", value="gesendete")
        n_contacts = st.number_input("Nr. von Kontakten", min_value=1, value=5)

        if kind_of_mail == "gesendete":
            sent = True
        else:
            sent = False

        # show contacts with count
        st.subheader(f"Kumulierte {kind_of_mail} E-Mail Kommunikation zwischen dem E-Mail Account und seinen Top {n_contacts} Kontakten")
        fig_contacts = acc.plot_communication_with_contacts(df_poi=df_poi, poi=poi, contacts_with_count=contacts_with_count, n_contacts=n_contacts, timeline=timeline, sent=sent)

        fig_contacts.set_figwidth(20)
        fig_contacts.set_figheight(10)

        st.pyplot(fig_contacts)

        st.session_state["is_analyzed"] = True

    # ------------------ settings for plot ----------------------
    if st.session_state["is_analyzed"]:
        st.subheader("Wähle einen Kontakt zu weiteren Analyse")
        # contacts = contacts_of_poi = acc.get_contacts_of_poi(df=df_poi, poi=poi)
        contact = st.selectbox("Wähle einen Kontakt:", contacts_with_count["contact"])
        cum_contact_time_series = acc.get_cumulated_contact_time_series(df_poi=df_poi, poi=poi, contact=contact, timeline=timeline)

        fig_contact, ax_contact = plt.subplots()

        fig_contact.set_figwidth(20)
        fig_contact.set_figheight(10)

        sns.lineplot(data=cum_contact_time_series, x=cum_contact_time_series.index, y=poi, label=poi, ax=ax_contact)
        sns.lineplot(data=cum_contact_time_series, x=cum_contact_time_series.index, y=contact, label=contact, ax=ax_contact)

        st.write("Kumulierte gesendete E-Mail Kommunikation zwischen dem Account und einem Kontakt")
        st.pyplot(fig_contact)

        st.session_state["plot_settings"] = True

    # ------------------ plot -----------------------------------

    if st.session_state["plot_settings"]:
        pass


import pygame
from pygame.sprite import Sprite

class Alien(Sprite):
    """Emulates and creates an alien ship"""

    def __init__(self, screen, settings):
        """Initialize the alien and set its starting position"""
        super().__init__()
        self.screen = screen
        self.settings = settings

        # Loads the alien image, scales it, then set its rect attribute
        self.image = pygame.image.load('images/alien_ship_grey.png')
        self.image = pygame.transform.scale(self.image, (int(self.settings.width/25), int(self.settings.width/25)))
        self.rect = self.image.get_rect()

        # Start alien ship at top left of screen
        self.rect.x = self.settings.width/10
        self.rect.y = self.settings.height/10

        self.x = float(self.rect.x)

    def check_edges(self):
        """Return True if alien is at edge of screen"""
        screen_rect = self.screen.get_rect()
        if self.rect.right >= screen_rect.right:
            return True
        elif self.rect.left <= 0:
            return True

    def update(self):
        """Updates position of alien"""
        self.x += self.settings.alien_speed_factor * self.settings.fleet_direction
        self.rect.x = self.x

    def blitme(self):
        """Draws the alien sprite"""
        self.screen.blit(self.image, self.rect)


import tensorflow as tf
import numpy as np
from tensorflow.examples.tutorials.mnist import input_data
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec

import os

class DenseLayer(object):
    def __init__(self, n_in, m_out, name, f=tf.nn.relu):
        W = tf.Variable(tf.random_normal(shape=(n_in, m_out)) * 2 / np.sqrt(n_in), name=name + "_W")
        b = tf.Variable(np.zeros(m_out).astype(np.float32), name=name + "_b")
        self.f = f

        self.W = W
        self.b = b
        self.params = [self.W, self.b]
        self.name = name

    def forward(self, X):
        a = tf.matmul(X, self.W) + self.b

        return self.f(a)

def plot_grid(samples):
    fig = plt.figure(figsize=(4,4))
    gs = gridspec.GridSpec(4,4)
    i = 0

    for sample in samples:
        if i > 15:
            break
        ax = plt.subplot(gs[i])
        plt.axis('off')
        i += 1

        ax.set_aspect('equal')

        plt.imshow(sample.reshape(28, 28), cmap='gray' )

    return fig

class GAN(object):
    def __init__(self, n_latent, d_sizes, g_sizes, index):

        self.index = index
        img_size = 28
        self.img_size = img_size
        self.d_sizes = d_sizes
        self.g_sizes = g_sizes
        self.g_layers = []
        self.d_layers = []

        self.n_latent = n_latent

        self.g_vars = []
        self.d_vars = []

        n_in = img_size**2

        self.g_sizes.append(img_size**2)
        self.d_sizes.append(1)
        mnist = input_data.read_data_sets('../data/MNIST_data', one_hot=True)
        self.X = mnist.train.images
        count = 0

        for d_size in self.d_sizes:
            name = "discriminator_" + str(count)
            if d_size != self.d_sizes[-1]:
                layer = DenseLayer(n_in, d_size, name)
            else:
                layer = DenseLayer(n_in, d_size, name, lambda x:x)

            self.d_layers.append(layer)
            self.d_vars.append(layer.params)
            count += 1
            n_in = d_size

        n_in = n_latent
        count = 0
        for g_size in self.g_sizes:
            name = "generator_" + str(count)

            if g_size != self.g_sizes[-1]:
                layer = DenseLayer(n_in, g_size, name)
            else:
                layer = DenseLayer(n_in, g_size, name, tf.nn.sigmoid)

            self.g_layers.append(layer)
            self.g_vars.append(layer.params)
            count += 1
            n_in = g_size

    def sample_generator(self, noise):
        # E = tf.placeholder(tf.float32, shape=(examples, self.n_latent))
        sample = noise
        for layer in self.g_layers:
            sample = layer.forward(sample)

        return sample

    def forward_discriminator(self, X):
        out = X
        for layer in self.d_layers:
            out = layer.forward(out)

        return out

    def fit(self):

        E = tf.placeholder(dtype = tf.float32, shape=(None, self.n_latent))
        Z = tf.placeholder(dtype=tf.float32, shape=(None, self.img_size**2))

        sampled_images = self.sample_generator(E)
        logits = self.forward_discriminator(Z)
        logits_sampled = self.forward_discriminator(sampled_images)
        sampled_images_test = self.sample_generator(E)

        d_cost_real = tf.nn.sigmoid_cross_entropy_with_logits(logits=logits, labels=tf.ones_like(logits))
        d_cost_fake = tf.nn.sigmoid_cross_entropy_with_logits(logits=logits_sampled, labels=tf.zeros_like(logits_sampled))

        g_cost = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=logits_sampled, labels=tf.ones_like(logits_sampled)))
        d_cost = tf.reduce_mean(d_cost_fake) + tf.reduce_mean(d_cost_real)

        d_train_op = tf.train.AdamOptimizer().minimize(d_cost, var_list=self.d_vars)
        g_train_op = tf.train.AdamOptimizer().minimize(g_cost, var_list=self.g_vars)

        self.init_op = tf.global_variables_initializer()
        self.sess = tf.InteractiveSession()
        self.sess.run(self.init_op)

        d_costs = []
        g_costs = []
        batch_sz = 16
        epochs = 1000

        n_batches = len(self.X) // batch_sz

        if not os.path.exists('../samples' + str(self.index) +'/'):
            os.makedirs('../samples' + str(self.index) +'/')

        k = 0
        for i in range(epochs):
            print("epoch:", i)
            np.random.shuffle(self.X)
            for j in range(n_batches):
                batch = self.X[j * batch_sz:(j + 1) * batch_sz]
                W = np.random.uniform(-1, 1, size=(batch_sz, self.n_latent))
                _, d_cost_value = self.sess.run((d_train_op, d_cost), feed_dict={Z: batch, E: W})
                _, g_cost_value = self.sess.run((g_train_op, g_cost), feed_dict={E: W})
                d_costs.append(d_cost_value)
                g_costs.append(g_cost_value)

                if j % 100 == 0:
                    print("epoch " + str(i) + " iter " + str(j) + " d cost: " + str(d_costs[-1]) + " g cost: " + str(g_costs[-1]))
                    Z_out_test = self.sess.run(tf.nn.sigmoid(sampled_images_test),
                                               feed_dict={E: W})

                    fig = plot_grid(Z_out_test)
                    plt.savefig('../samples' + str(self.index) +'/' + str(k)+ '.png')
                    k += 1
                    plt.close(fig)

g = GAN(50, [150], [300], 1)
g.fit()

g2 = GAN(30, [150, 500], [150, 500], 2)
g2.fit()


from contextvars import ContextVar
from typing import Any, Optional

from sanic.log import logger
from sqlalchemy.ext.asyncio.engine import AsyncEngine, create_async_engine
from sqlalchemy.ext.asyncio.session import AsyncSession
from sqlalchemy.orm import relationship, selectinload
from sqlalchemy.orm.exc import UnmappedClassError
from sqlalchemy.orm.mapper import class_mapper
from sqlalchemy.sql.expression import select

from heliotrope.abc.database import AbstractGalleryinfoDatabase
from heliotrope.database.orm.base import mapper_registry
from heliotrope.database.orm.table import *
from heliotrope.domain import *

_base_model_session_ctx: ContextVar[AsyncSession] = ContextVar("session")

class _SessionManager:
    def __init__(self, engine: AsyncEngine) -> None:
        self.engine = engine

    async def __aenter__(self) -> "_SessionManager":
        self.session = AsyncSession(self.engine, expire_on_commit=False)
        self.session_ctx_token = _base_model_session_ctx.set(self.session)
        return self

    async def __aexit__(self, *args: Any) -> None:
        if hasattr(self, "session_ctx_token"):
            _base_model_session_ctx.reset(self.session_ctx_token)
            await self.session.close()

class ORM(AbstractGalleryinfoDatabase):
    def __init__(self, engine: AsyncEngine) -> None:
        self.engine = engine

    @staticmethod
    def mapping() -> None:
        mapper_registry.map_imperatively(
            Galleryinfo,
            galleryinfo_table,
            properties={
                "files": relationship(File),
                "tags": relationship(Tag),
                "artists": relationship(Artist),
                "characters": relationship(Character),
                "groups": relationship(Group),
                "parodys": relationship(Parody),
                "scene_indexes": relationship(SceneIndex),
                "languages": relationship(Language),
                "related": relationship(Related),
            },
        )
        mapper_registry.map_imperatively(Tag, tag_table)
        mapper_registry.map_imperatively(File, file_table)
        mapper_registry.map_imperatively(Artist, artist_table)
        mapper_registry.map_imperatively(Character, character_table)
        mapper_registry.map_imperatively(Group, group_table)
        mapper_registry.map_imperatively(Parody, parody_table)
        mapper_registry.map_imperatively(SceneIndex, scene_index_table)
        mapper_registry.map_imperatively(Related, related_table)
        mapper_registry.map_imperatively(Language, language_table)
        return None

    @classmethod
    async def setup(cls, db_url: str) -> "ORM":
        logger.debug(f"Setting up {cls.__name__}")
        # Because it is mapped manually, an error occurs if it is already mapped.
        # 수동으로 매핑하기때문에 이미 매핑돼있을경우 오류가 발생합니다.
        try:
            class_mapper(Galleryinfo)
        except UnmappedClassError:
            cls.mapping()
        engine = create_async_engine(db_url)
        async with engine.begin() as connection:
            await connection.run_sync(
                mapper_registry.metadata.create_all, checkfirst=True
            )
        return cls(engine)

    async def add_galleryinfo(self, galleryinfo: Galleryinfo) -> None:
        async with _SessionManager(self.engine) as manager:
            async with manager.session.begin():
                manager.session.add(galleryinfo)

        return None

    async def get_galleryinfo(self, id: int) -> Optional[Galleryinfo]:
        if -2147483648 < id > 2147483647:
            return None
        async with _SessionManager(self.engine) as manager:
            async with manager.session.begin():
                r = await manager.session.get(
                    Galleryinfo,
                    id,
                    [
                        selectinload(Galleryinfo.files),
                        selectinload(Galleryinfo.tags),
                        selectinload(Galleryinfo.artists),
                        selectinload(Galleryinfo.characters),
                        selectinload(Galleryinfo.groups),
                        selectinload(Galleryinfo.parodys),
                        selectinload(Galleryinfo.scene_indexes),
                        selectinload(Galleryinfo.languages),
                        selectinload(Galleryinfo.related),
                    ],
                )
                if r:
                    return r

        return None

    async def get_all_index(self) -> list[int]:
        async with _SessionManager(self.engine) as manager:
            async with manager.session.begin():
                result = await manager.session.execute(select(Galleryinfo.id))
                return [int(id) for id, in result.all()]


import json
import re
import unicodedata

import sqlalchemy
from sqlalchemy import text

import db
import db.dataset_eval
from db import exceptions
from utils import dataset_validator

def slugify(string):
    """Converts unicode string to lowercase, removes alphanumerics and
    underscores, and converts spaces to hyphens. Also strips leading and
    trailing whitespace.
    """
    string = unicodedata.normalize('NFKD', string).encode('ascii', 'ignore').decode('ascii')
    string = re.sub('[^\w\s-]', '', string).strip().lower()
    return re.sub('[-\s]+', '-', string)

def create_from_dict(dictionary, author_id):
    """Creates a new dataset from a dictionary.

    Returns:
        Tuple with two values: new dataset ID and error. If error occurs first
        will be None and second is an exception. If there are no errors, second
        value will be None.
    """
    dataset_validator.validate(dictionary)

    with db.engine.begin() as connection:
        if "description" not in dictionary or dictionary["description"] is None:
            dictionary["description"] = ""

        result = connection.execute("""INSERT INTO dataset (id, name, description, public, author)
                          VALUES (uuid_generate_v4(), %s, %s, %s, %s) RETURNING id""",
                                    (dictionary["name"], dictionary["description"], dictionary["public"], author_id))
        dataset_id = result.fetchone()[0]

        for cls in dictionary["classes"]:
            if "description" not in cls or cls["description"] is None:
                cls["description"] = ""
            result = connection.execute("""INSERT INTO dataset_class (name, description, dataset)
                              VALUES (%s, %s, %s) RETURNING id""",
                                        (cls["name"], cls["description"], dataset_id))
            cls_id = result.fetchone()[0]

            # Remove duplicate recordings, preserving order
            seen = set()
            cls["recordings"] = [r for r in cls["recordings"] if not (r in seen or seen.add(r))]

            for recording_mbid in cls["recordings"]:
                connection.execute("INSERT INTO dataset_class_member (class, mbid) VALUES (%s, %s)",
                                   (cls_id, recording_mbid))

    return dataset_id

def update_dataset_meta(dataset_id, meta):
    """ Update the metadata (name, description, public) for a dataset.

    Args:
        dataset_id: id of the dataset to update
        meta: dictionary of metadata to update

    Valid keys for `meta` are: `name`, `description`, `public`.
    If one of these keys is not present, the corresponding field
    for the dataset will not be updated. If no keys are present,
    the dataset will not be updated.
    """

    params = {}
    updates = []
    # Check meta values against None because "" and False are valid values
    name = meta.pop("name", None)
    if name is not None:
        updates.append("name = :name")
        params["name"] = name
    desc = meta.pop("description", None)
    if desc is not None:
        updates.append("description = :description")
        params["description"] = desc
    public = meta.pop("public", None)
    if public is not None:
        updates.append("public = :public")
        params["public"] = public

    if meta:
        raise ValueError("Unexpected meta value(s): %s" % ", ".join(meta.keys()))

    setstr = ", ".join(updates)
    query = text("""UPDATE dataset
                       SET %s
                     WHERE id = :id
            """ % setstr)

    # Only do an update if we have items to update
    if params:
        params["id"] = dataset_id
        with db.engine.begin() as connection:
            connection.execute(query, params)

def update(dataset_id, dictionary, author_id):
    # TODO(roman): Make author_id argument optional (keep old author if None).
    dataset_validator.validate(dictionary)

    with db.engine.begin() as connection:
        if "description" not in dictionary or dictionary["description"] is None:
            dictionary["description"] = ""

        connection.execute("""UPDATE dataset
                          SET (name, description, public, author, last_edited) = (%s, %s, %s, %s, now())
                          WHERE id = %s""",
                           (dictionary["name"], dictionary["description"], dictionary["public"], author_id, dataset_id))

        # Replacing old classes with new ones
        connection.execute("""DELETE FROM dataset_class WHERE dataset = %s""", (dataset_id,))

        for cls in dictionary["classes"]:
            if "description" not in cls or cls["description"] is None:
                cls["description"] = ""
            result = connection.execute("""INSERT INTO dataset_class (name, description, dataset)
                              VALUES (%s, %s, %s) RETURNING id""",
                                        (cls["name"], cls["description"], dataset_id))
            cls_id = result.fetchone()[0]

            for recording_mbid in cls["recordings"]:
                connection.execute("INSERT INTO dataset_class_member (class, mbid) VALUES (%s, %s)",
                                   (cls_id, recording_mbid))

def get(id):
    """Get dataset with a specified ID.

    Returns:
        Dictionary with dataset details if it has been found, None
        otherwise.
    """
    with db.engine.connect() as connection:
        result = connection.execute(
            "SELECT id::text, name, description, author, created, public, last_edited "
            "FROM dataset "
            "WHERE id = %s",
            (str(id),)
        )
        if result.rowcount < 1:
            raise exceptions.NoDataFoundException("Can't find dataset with a specified ID.")
        row = dict(result.fetchone())
        row["classes"] = _get_classes(row["id"])
        row["num_recordings"] = sum([len(cl.get("recordings", [])) for cl in row["classes"]])
        return row

def get_public_datasets(status):
    if status == "all":
        statuses = db.dataset_eval.VALID_STATUSES
    elif status in db.dataset_eval.VALID_STATUSES:
        statuses = [status]
    else:
        raise ValueError("Unknown status")

    with db.engine.connect() as connection:
        query = text("""
            SELECT dataset.id::text
                 , dataset.name
                 , dataset.description
                 , "user".musicbrainz_id AS author_name
                 , dataset.created
                 , job.status
            FROM dataset
            JOIN "user"
              ON "user".id = dataset.author
              LEFT JOIN LATERAL (SELECT dataset_eval_jobs.status
                                   FROM dataset_eval_jobs
                                   JOIN dataset_snapshot
                                     ON dataset_snapshot.id = dataset_eval_jobs.snapshot_id
                                  WHERE dataset.id = dataset_snapshot.dataset_id
                               ORDER BY dataset_eval_jobs.updated DESC
                               LIMIT 1)
                               AS JOB ON TRUE
          WHERE dataset.public = 't'
            AND job.status = ANY((:status)::eval_job_status[])
       ORDER BY dataset.created DESC
             """)
        result = connection.execute(query, {"status": statuses})
        return [dict(row) for row in result]

def _get_classes(dataset_id):
    with db.engine.connect() as connection:
        query = text("""SELECT id::text
                             , name
                             , description
                          FROM dataset_class
                         WHERE dataset = :dataset_id
                      ORDER BY id
        """)
        result = connection.execute(query, {"dataset_id": dataset_id})
        rows = result.fetchall()
        classes = []
        for row in rows:
            row = dict(row)
            row["recordings"] = _get_recordings_in_class(row["id"])
            classes.append(row)
        return classes

def _get_recordings_in_class(class_id):
    with db.engine.connect() as connection:
        result = connection.execute("SELECT mbid::text FROM dataset_class_member WHERE class = %s",
                                    (class_id,))
        recordings = []
        for row in result:
            recordings.append(row["mbid"])
        return recordings

def get_by_user_id(user_id, public_only=True):
    """Get datasets created by a specified user.

    Returns:
        List of dictionaries with dataset details.
    """
    with db.engine.connect() as connection:
        where = "WHERE author = %s"
        if public_only:
            where += " AND public = TRUE"
        result = connection.execute("SELECT id, name, description, author, created "
                                    "FROM dataset " + where,
                                    (user_id,))
        datasets = []
        for row in result:
            datasets.append(dict(row))
        return datasets

def delete(id):
    """Delete dataset with a specified ID."""
    with db.engine.begin() as connection:
        connection.execute("DELETE FROM dataset WHERE id = %s", (str(id),))

def create_snapshot(dataset_id):
    """Creates a snapshot of current version of a dataset.

    Snapshots are stored as JSON and have the following structure:
    {
        "name": "..",
        "description": "..",
        "classes": [
            {
                "name": "..",
                "description: "..",
                "recordings": ["..", ...]
            },
            ...
        ]
    }

    Args:
        dataset_id (string/uuid): ID of a dataset.

    Returns:
        ID (UUID) of a snapshot that was created.
    """
    dataset = get(dataset_id)
    if not dataset:
        raise exceptions.NoDataFoundException("Can't find dataset with a specified ID.")
    snapshot = {
        "name": dataset["name"],
        "description": dataset["description"],
        "classes": [{
                        "name": c["name"],
                        "description": c["description"],
                        "recordings": c["recordings"],
                    } for c in dataset["classes"]],
    }
    with db.engine.connect() as connection:
        result = connection.execute(sqlalchemy.text("""
            INSERT INTO dataset_snapshot (id, dataset_id, data)
                 VALUES (uuid_generate_v4(), :dataset_id, :data)
              RETURNING id::text
        """), {
            "dataset_id": dataset_id,
            "data": json.dumps(snapshot),
        })
        return result.fetchone()["id"]

def get_snapshot(id):
    """Get snapshot of a dataset.

    Args:
        id (string/uuid): ID of a snapshot.

    Returns:
        dictionary: {
            "id": <ID of the snapshot>,
            "dataset_id": <ID of the dataset that this snapshot is associated with>,
            "created": <creation time>,
            "data": <actual content of a snapshot (see `create_snapshot` function)>
        }
    """
    with db.engine.connect() as connection:
        result = connection.execute(sqlalchemy.text("""
            SELECT id::text
                 , dataset_id::text
                 , data
                 , created
              FROM dataset_snapshot
             WHERE id = :id
        """), {"id": id})
        row = result.fetchone()
        if not row:
            raise db.exceptions.NoDataFoundException("Can't find dataset snapshot with a specified ID.")
        return dict(row)

def _delete_snapshot(connection, snapshot_id):
    """Delete a snapshot.

    Args:
        connection: an SQLAlchemy connection.
        snapshot_id (string/uuid): ID of a snapshot.
    """
    query = sqlalchemy.text("""
        DELETE FROM dataset_snapshot
              WHERE id = :snapshot_id""")
    connection.execute(query, {"snapshot_id": snapshot_id})

def get_snapshots_for_dataset(dataset_id):
    """Get all snapshots created for a dataset.

    Args:
        dataset_id (string/uuid): ID of a dataset.

    Returns:
        List of snapshots as dictionaries.
    """
    with db.engine.connect() as connection:
        result = connection.execute(sqlalchemy.text("""
            SELECT id::text
                 , dataset_id::text
                 , data
                 , created
              FROM dataset_snapshot
             WHERE dataset_id = :dataset_id
        """), {"dataset_id": dataset_id})
        return [dict(row) for row in result]

def _delete_snapshots_for_dataset(connection, dataset_id):
    """Delete all snapshots of a dataset.

    Args:
        connection: an SQLAlchemy connection.
        dataset_id (string/uuid): ID of a dataset.
    """
    query = sqlalchemy.text("""
        DELETE FROM dataset_snapshot
              WHERE dataset_id = :dataset_id""")
    connection.execute(query, {"dataset_id": dataset_id})

def _get_classid_for_dataset(connection, dataset_id, class_name):
    query = sqlalchemy.text("""
      SELECT id
        FROM dataset_class
       WHERE name = :name
         AND dataset = :dataset_id""")
    result = connection.execute(query, {"name": class_name, "dataset_id": dataset_id})
    if result.rowcount < 1:
        raise exceptions.NoDataFoundException("No such class exists.")
    clsid = result.fetchone()
    return clsid[0]

def add_recordings(dataset_id, class_name, recordings):
    """Adds new recordings to a class in a dataset.

    If any given recording already exist in this class it is not added again.

    Args:
        dataset_id: the uuid of the dataset
        class_name: The class to add the recordings to
        recordings: List of recordings (MBID strings) to add

    Raises:
        NoDataFoundException if the dataset doesn't exist or if the class doesn't exist in the dataset
    """

    with db.engine.begin() as connection:
        clsid = _get_classid_for_dataset(connection, dataset_id, class_name)
        for mbid in recordings:
            connection.execute(sqlalchemy.text("""
              INSERT INTO dataset_class_member (class, mbid)
                   SELECT :clsid, :mbid
         WHERE NOT EXISTS (SELECT *
                             FROM dataset_class_member d
                            WHERE d.class = :clsid
                              AND d.mbid = :mbid)
                      """),
                               {"clsid": clsid, "mbid": mbid})

def delete_recordings(dataset_id, class_name, recordings):
    """Delete recordings from a dataset class"""

    with db.engine.begin() as connection:
        clsid = _get_classid_for_dataset(connection, dataset_id, class_name)
        for mbid in recordings:
            connection.execute(sqlalchemy.text("""
                DELETE FROM dataset_class_member
                      WHERE class = :class_name
                        AND mbid = :mbid_num
                            """),
                               {"class_name": clsid, "mbid_num": mbid})

def add_class(dataset_id, class_data):
    """Add a class to a dataset

    If the dict argument contains a key "recordings", add these recordings
    to the class as well. See :func:`add_recordings`.
    If the class already exists, do not add it again, but add recordings
    to it if they are given.

    Args:
        dataset_id: the UUID of the dataset to add this class to
        class_data: A dictionary representing the class to add:
              {"name": "Classname", "description": "Class desc",
               "recordings": [list of recording ids (optional)}
    """

    with db.engine.begin() as connection:
        if "description" not in class_data:
            class_data["description"] = ""
        connection.execute(sqlalchemy.text("""
            INSERT INTO dataset_class (name, description, dataset)
                 SELECT :name, :description, :datasetid
       WHERE NOT EXISTS (SELECT *
                           FROM dataset_class d
                          WHERE d.name = :name
                            AND d.dataset = :datasetid)
                    """),
                           {"name": class_data["name"], "description": class_data["description"], "datasetid": dataset_id})
    if "recordings" in class_data:
        add_recordings(dataset_id, class_data["name"], class_data["recordings"])

def delete_class(dataset_id, class_data):
    """Delete a class from a dataset

    Deletes the class members as well since we have ON DELETE CASCADE set in the database

    Args:
        dataset_id: the UUID of the dataset to delete this class from
        class_data: A dictionary representing the class to delete:
              {"name": "Classname"}
    """

    with db.engine.begin() as connection:
        query = sqlalchemy.text("""
            DELETE FROM dataset_class
                  WHERE name = :class_name
                    AND dataset = :dataset_id
                        """)
        connection.execute(query,  {"class_name": class_data["name"], "dataset_id": dataset_id})

def update_class(dataset_id, class_name, class_data):
    """ Update the metadata (name, description) for a class.

    Args:
        dataset_id: id of the dataset to update
        class_name: the name of the class to update
        class_data: dictionary of metadata to update

    Valid keys for `meta` are: `name` (required),
     `new_name` (optional), `description` (optional)
    If one of the optional keys is not present, the corresponding field
    for the class will not be updated. If no keys are present,
    the class will not be updated.
    """

    params = {}
    updates = []
    if "new_name" in class_data:
        updates.append("name = :new_name")
        params["new_name"] = class_data["new_name"]
    if "description" in class_data:
        updates.append("description = :description")
        params["description"] = class_data["description"]
    setstr = ", ".join(updates)
    update_query = text("""UPDATE dataset_class
                               SET %s
                             WHERE id = :id

            """ % setstr)

    if params:
        with db.engine.begin() as connection:
            clsid = _get_classid_for_dataset(connection, dataset_id, class_name)
            params["id"] = clsid

            connection.execute(update_query, params)

def check_recording_in_dataset(dataset_id, mbid):
    """Check whether an MBID is in a given dataset.
    Args:
        dataset_id: ID of a dataset
        mbid: MBID to be checked
    Returns:
        True if an MBID appears anywhere in a dataset, False otherwise
    """
    with db.engine.connect() as connection:
        result = connection.execute(sqlalchemy.text("""
            SELECT dataset_class.id
              FROM dataset_class
              JOIN dataset_class_member
                ON dataset_class_member.class = dataset_class.id
             WHERE dataset_class.dataset = :dataset_id
               AND dataset_class_member.mbid = :mbid
        """), {"dataset_id": dataset_id, "mbid": mbid})

        return result.rowcount > 0


import numpy as np
import cv2 as cv
img = cv.imread('coins.jpg', 0)
img = cv.medianBlur(img, 5)
cimg = cv.cvtColor(img, cv.COLOR_GRAY2BGR)
#  get circles of 20-100 pixels in diameter
circles = cv.HoughCircles(
    img, cv.HOUGH_GRADIENT, 1, 65, param1=50, param2=60, minRadius=120, maxRadius=200)

circles = np.uint16(np.around(circles))
for i in circles[0, :]:
    # draw the outer circle
    cv.circle(cimg, (i[0], i[1]), i[2], (0, 255, 0), 2)
    # draw the center of the circle
    cv.circle(cimg, (i[0], i[1]), 2, (0, 0, 255), 3)
cv.imshow('detected circles', cimg)
cv.waitKey(0)
cv.destroyAllWindows()

#  read the following images from the same folder
#  2, 5,50c, 50old, 10, 1

coin2 = cv.imread('2.png', 0)
coin5 = cv.imread('5.png', 0)
coin50c = cv.imread('50c.png', 0)
coin50old = cv.imread('50old.png', 0)
coin10 = cv.imread('10.png', 0)
coin1 = cv.imread('1.png', 0)

# #  using SIFT to find the most similar coin in the iamge and mark it with is name

sift = cv.xfeatures2d.SIFT_create()
kp1, des1 = sift.detectAndCompute(img, None)
kp2, des2 = sift.detectAndCompute(coin2, None)
kp3, des3 = sift.detectAndCompute(coin5, None)
kp4, des4 = sift.detectAndCompute(coin50c, None)
kp5, des5 = sift.detectAndCompute(coin50old, None)
kp6, des6 = sift.detectAndCompute(coin10, None)
kp7, des7 = sift.detectAndCompute(coin1, None)

#  using FLANN to match the coins
FLANN_INDEX_KDTREE = 5
index_params = dict(algorithm=FLANN_INDEX_KDTREE, trees=10)
search_params = dict(checks=60)
flann = cv.FlannBasedMatcher(index_params, search_params)
matches2 = flann.knnMatch(des1, des2, k=2)
matches5 = flann.knnMatch(des1, des3, k=2)
matches50c = flann.knnMatch(des1, des4, k=2)
matches50old = flann.knnMatch(des1, des5, k=2)
matches10 = flann.knnMatch(des1, des6, k=2)
matches1 = flann.knnMatch(des1, des7, k=2)

#  store the good matches as per Lowe's ratio test.
good2 = []
good5 = []
good50c = []
good50old = []
good10 = []
good1 = []
for m, n in matches2:
    if m.distance < 0.7 * n.distance:
        good2.append(m)
for m, n in matches5:
    if m.distance < 0.7 * n.distance:
        good5.append(m)
for m, n in matches50c:
    if m.distance < 0.7 * n.distance:
        good50c.append(m)
for m, n in matches50old:
    if m.distance < 0.7 * n.distance:
        good50old.append(m)
for m, n in matches10:
    if m.distance < 0.7 * n.distance:
        good10.append(m)
for m, n in matches1:
    if m.distance < 0.7 * n.distance:
        good1.append(m)

 #  draw the matches  only if the number of matches is greater than 10

img3 = cv.drawMatches(img, kp1, coin2, kp2, good2, None, flags=2)
img4 = cv.drawMatches(img, kp1, coin5, kp3, good5, None, flags=2)
img5 = cv.drawMatches(img, kp1, coin50c, kp4, good50c, None, flags=2)
img6 = cv.drawMatches(img, kp1, coin50old, kp5, good50old, None, flags=2)
img7 = cv.drawMatches(img, kp1, coin10, kp6, good10, None, flags=2)
img8 = cv.drawMatches(img, kp1, coin1, kp7, good1, None, flags=2)

#  show the images
cv.imshow('2', img3)
cv.imshow('5', img4)
cv.imshow('50c', img5)
cv.imshow('50old', img6)
cv.imshow('10', img7)
cv.imshow('1', img8)
cv.waitKey(0)
cv.destroyAllWindows()


import os
import sys
sys.path.insert(0, os.getcwd())

os.environ.setdefault("DJANGO_SETTINGS_MODULE", "price_comparison_site.settings")

import django
django.setup()

from search import two

from django.shortcuts import render
from search.models import Product

import numpy as np
import pandas as pd
from django_pandas.io import read_frame

from transformers import TFBertForMaskedLM
from transformers import AutoTokenizer

from collections import Counter

# Create your views here.
def index(request):
    return render(request, 'search/index.html')

def search(request):
    return render(request, 'search/main.html')

def result(request, q):
    request.q_value = q
    print(request.session.get('query'))
    print(q)
    if request.session.get('query') != q:
        request.session['scraped'] = False
    if request.session.get('scraped') != True:
        Product.objects.all().delete()
        two.coupang(q)
        request.session['scraped'] = True
        request.session['query'] = q
    context = {
        'q': q,
        'page': 1,
        'result': Product.objects.all()[:20],
        'length': len(Product.objects.all()),
    }
    return render(request, 'search/result.html', context)

def result2(request, q, page):
    request.q_value = q
    print(request.session.get('query'))
    print(q)
    if request.session.get('query') != q:
        request.session['scraped'] = False
    if request.session.get('scraped') != True:
        Product.objects.all().delete()
        two.coupang(q)
        request.session['scraped'] = True
        request.session['query'] = q
    try:
        if page > len(Product.objects.all()) / 20:
            raise SyntaxError('')
        context = {
            'q': q,
            'page': page,
            'result': Product.objects.all()[(20*(page-1)):((20*(page-1))+20)],
            'length': len(Product.objects.all()),
        }
    except:
        context = {
            'q': q,
            'page': int(len(Product.objects.all())/20)+1,
            'result': Product.objects.all()[len(Product.objects.all())-20:len(Product.objects.all())],
            'length': len(Product.objects.all()),
        }
    finally:
        return render(request, 'search/result.html', context)

def result3(request, q):
    request.q_value = q

    qs = Product.objects.all()
    df = read_frame(qs)

    try:
        minstar = request.GET.get('minstar')
        if minstar is None:
            minstar = 0
        else:
            minstar = float(minstar)
    except:
        minstar = 0
    finally:
        pass

    try:
        minprice = request.GET.get('minprice')
        if minprice is None:
            minprice = 0
        else:
            minprice = int(minprice)
    except:
        minprice = 0
    finally:
        pass

    try:
        maxprice = request.GET.get('maxprice')
        if maxprice is None:
            maxprice = 2147483647
        else:
            maxprice = int(maxprice)
    except:
        maxprice = 2147483647
    finally:
        pass

    df2 = df.loc[(pd.to_numeric(df['price']) >= int(minprice)) & (pd.to_numeric(df['price']) <= int(maxprice)) & (pd.to_numeric(df['star']) >= float(minstar))]

    detail = request.GET.get('detail')
    if detail:
        model = TFBertForMaskedLM.from_pretrained('klue/bert-base', from_pt=True)
        tokenizer = AutoTokenizer.from_pretrained("klue/bert-base")
        token1 = Counter(tokenizer(detail)['input_ids'])
        similarity = []
        for idx in df2.index:
            token2 = Counter(tokenizer(df2.loc[idx,'product_name'])['input_ids'])
            intersection = sum((token1 & token2).values())
            union = sum((token1 | token2).values())
            similarity.append(intersection / union if union > 0 else 0.0)
        df2['sim'] = similarity
        df2 = df2.sort_values(by='sim',ascending=False)

        print(df2[:20].to_dict(orient='index'))

    context = {
        'q': q,
        'page': 1,
        'result': df2[:20].to_dict(orient='records'),
        'length': len(df2),
    }
    return render(request, 'search/detail.html', context)


#create dictonary of parameter

import requests
#just returning whatever we send
param={'name': 'testing', 'email': 'test@gmail.com', 'number': '34343433'}
response = requests.get("https://httpbin.org/get", params=param)
print(response.text)

#pytest unit test framework
#why need?
#many adva (i) in which condition what all test case can be executed
#(ii) prerequisite (before) and post script in unit test framework (after execution)
#(iii) assertions - actual and expected result
#(iv) generate report