import asyncio
import gc
import os
from joblib import Parallel, delayed
import pandas as pd
from prefect import flow, task
from prefect_ray.task_runners import RayTaskRunner
import numpy as np
from numpy import array, float32  # to eval strings as arrays

from sklearn.metrics import log_loss

import ray
from sklearn.model_selection import StratifiedKFold

from mlStack import (
    bootstrap,
    remove_all_flows,
    main as runMLstack,
    serializeBootstrapResults,
    serializeResultsDataframe,
)
from metaconfig import metaconfig
from config import config
from models import stack as modelStack
from tasks.input import processInputFiles

def relativePerplexity(y_true, y_pred, y_true_baseline, y_pred_baseline, epsilon=1e-15):
    samplePerplexity = perplexity(y_true, y_pred)
    baselineSamplePerplexity = perplexity(y_true_baseline, y_pred_baseline)

    return pd.Series(
        [
            samplePerplexity,
            baselineSamplePerplexity,
            np.divide(samplePerplexity, baselineSamplePerplexity + epsilon),
        ]
    )  # relative perplexity = perplexity / perplexity of model with single-most case correlated feature

def getBaselineFeatureResults(
    caseGenotypes,
    controlGenotypes,
    holdoutCaseGenotypes,
    holdoutControlGenotypes,
    clinicalData,
    config,
):
    selectedFeature = findBaselineFeature(caseGenotypes, controlGenotypes)
    outerCvIterator = StratifiedKFold(
        n_splits=config["sampling"]["crossValIterations"], shuffle=False
    )
    innerCvIterator = outerCvIterator

    bootstrap_args = [
        (
            caseGenotypes.loc[[selectedFeature]],
            controlGenotypes.loc[[selectedFeature]],
            holdoutCaseGenotypes.loc[[selectedFeature]],
            holdoutControlGenotypes.loc[[selectedFeature]],
            clinicalData,
            model,
            hyperParameterSpace,
            innerCvIterator,
            outerCvIterator,
            config,
            False,  # disable tracking
        )
        for model, hyperParameterSpace in list(modelStack.items())
    ]
    results = Parallel(n_jobs=-1)(delayed(bootstrap)(*args) for args in bootstrap_args)

    baselineFeatureResults = {}
    for i in range(len(modelStack)):
        modelResults = results[i]
        baselineFeatureResults = serializeBootstrapResults(
            modelResults, baselineFeatureResults
        )
    baselineFeatureResultsDataframe = serializeResultsDataframe(baselineFeatureResults)
    baselineFeatureResultsDataframe.index.name = "id"

    return baselineFeatureResultsDataframe, selectedFeature

def perplexity(y_true, y_pred):
    crossEntropy = log_loss(
        y_true, y_pred, labels=[0, 1], eps=1e-15
    )  # linear predictions (exactly 0 or 1) depend on offset of 1e-15 when log is applied to avoid NaN

    # perplexity = 2 ^ crossEntropy
    return np.power(2, crossEntropy)

def findBaselineFeature(caseGenotypes, controlGenotypes):
    # calculate the mean of each feature for cases and controls
    mean_cases = caseGenotypes.mean(axis=1)
    mean_controls = controlGenotypes.mean(axis=1)

    # calculate the absolute difference in means for each feature
    diff_means = abs(mean_cases - mean_controls)

    # get the feature with the largest difference in means
    selected_feature = diff_means.idxmax()

    print("Selected feature for baseline perplexity: ", selected_feature)
    return selected_feature

@task(retries=10)
def measureIterations(
    result,
    caseGenotypes,
    controlGenotypes,
    holdoutCaseGenotypes,
    holdoutControlGenotypes,
    clinicalData,
):
    baselineFeatureResults, selectedFeature = getBaselineFeatureResults(
        caseGenotypes,
        controlGenotypes,
        holdoutCaseGenotypes,
        holdoutControlGenotypes,
        clinicalData,
        config,
    )
    # serialize probability arrays from string
    result["probability"] = result["probability"].apply(lambda x: np.array(eval(x)))
    # take intersection of bootstrapped samples
    result = result.loc[baselineFeatureResults.index.intersection(result.index)]
    baselineFeatureResults = baselineFeatureResults.loc[result.index]
    result["baselineProbability"] = baselineFeatureResults["probability"]

    relativePerplexities = pd.DataFrame(index=result.index)
    new_cols = result.apply(
        lambda row: relativePerplexity(
            [row["label"]] * len(row["probability"]),
            row["probability"],
            [row["label"]] * len(row["baselineProbability"]),
            row["baselineProbability"],
        ),
        axis=1,
        result_type="expand",
    )

    (
        relativePerplexities["all features"],
        relativePerplexities[f"{selectedFeature}"],
        relativePerplexities["relative"],
    ) = (new_cols[0], new_cols[1], new_cols[2])

    # separate discordant & well-classified
    discordantSamples = result.loc[
        result["accuracy"] <= metaconfig["samples"]["discordantThreshold"],
    ]
    accurateSamples = result.loc[
        result["accuracy"] >= metaconfig["samples"]["accurateThreshold"],
    ]

    return (
        relativePerplexities,
        discordantSamples,
        accurateSamples,
        baselineFeatureResults,
    )

def readSampleIDs(table, label):
    sampleIDs = table.loc[table["label"] == label]
    return sampleIDs.index.tolist()

def sequesterOutlierSamples():
    samples = {
        "accurate": pd.read_csv(
            f"projects/{config['tracking']['project']}/accurateSamples.csv",
            index_col="id",
        ),
        "discordant": pd.read_csv(
            f"projects/{config['tracking']['project']}/discordantSamples.csv",
            index_col="id",
        ),
    }

    for sampleType, dataframe in samples.items():
        for labelType, label in {"case": 1, "control": 0}.items():
            if metaconfig["samples"]["sequester"][sampleType][labelType]:
                ids = readSampleIDs(dataframe, label=label)
                config["sampling"]["sequesteredIDs"].extend(ids)

@flow()
def main():
    newWellClassified = True
    countSuffix = 1

    baseProjectPath = config["tracking"]["project"]
    while newWellClassified:
        gc.collect()
        config["tracking"]["project"] = f"{baseProjectPath}__{str(countSuffix)}"
        if countSuffix <= metaconfig["tracking"]["lastIteration"]:
            sequesterOutlierSamples()
            countSuffix += 1
            continue
        else:
            (
                results,
                genotypeData,
                clinicalData,
            ) = runMLstack(config)
            config["sampling"]["lastIteration"] = 0
        currentResults = pd.read_csv(
            f"projects/{config['tracking']['project']}/sampleResults.csv",
            index_col="id",
        )
        (
            samplePerplexities,
            discordantSamples,
            accurateSamples,
            baselineFeatureResults,
        ) = measureIterations(
            currentResults,
            genotypeData.case.genotype,
            genotypeData.control.genotype,
            genotypeData.holdout_case.genotype,
            genotypeData.holdout_control.genotype,
            clinicalData,
        )

        samplePerplexities["accuracy"] = currentResults.loc[
            currentResults.index.intersection(samplePerplexities.index)
        ]["accuracy"]
        samplePerplexities["baselineAccuracy"] = baselineFeatureResults.loc[
            baselineFeatureResults.index.intersection(samplePerplexities.index)
        ]["accuracy"]
        samplePerplexities["label"] = currentResults.loc[
            currentResults.index.intersection(samplePerplexities.index)
        ]["label"]

        np.set_printoptions(threshold=np.inf)
        samplePerplexities.to_csv(
            f"projects/{config['tracking']['project']}/samplePerplexities.csv"
        )
        accurateSamples.to_csv(
            f"projects/{config['tracking']['project']}/accurateSamples.csv"
        )
        discordantSamples.to_csv(
            f"projects/{config['tracking']['project']}/discordantSamples.csv"
        )
        baselineFeatureResults["probability"] = baselineFeatureResults[
            "probability"
        ].map(lambda x: np.array2string(np.array(x), separator=","))
        baselineFeatureResults.to_csv(
            f"projects/{config['tracking']['project']}/baselineFeatureSampleResults.csv"
        )
        if (
            len(
                currentResults.loc[
                    (currentResults["label"] == 1)
                    & (
                        currentResults["accuracy"]
                        >= metaconfig["samples"]["accurateThreshold"]
                    )
                ]
            )
            == 0
        ):
            newWellClassified = False
            break
        sequesterOutlierSamples()
        countSuffix += 1

if __name__ == "__main__":
    # TODO replace notebook code with src imports
    ray.shutdown()
    main()
    clearHistory = True
    if clearHistory:
        asyncio.run(remove_all_flows())


#!/usr/bin/python
# -*- coding: utf8 -*-

import json
import boto3
import time
import os

TABLE_NAME = os.getenv('dynamodb_table')
WEBSITE_BUCKET = os.getenv('bucket')

def update_json_file_to_s3():
    client = boto3.client('dynamodb')

    response = client.scan(
        TableName = TABLE_NAME,
        Select = 'SPECIFIC_ATTRIBUTES',
        ProjectionExpression = 'service, #r, #s',
        ExpressionAttributeNames = {
            "#r": "region",
            "#s": "status"
        }
    )
    flag = response.has_key('LastEvaluatedKey')
    items = response['Items']
    count = response['Count']

    while (flag):
        response = client.scan(
            TableName = TABLE_NAME,
            Select = 'SPECIFIC_ATTRIBUTES',
            ProjectionExpression = 'service, #r, #s',
            ExpressionAttributeNames = {
                "#r": "region",
                "#s": "status"
            },
            ExclusiveStartKey = response['LastEvaluatedKey']
        )

        items = items + response['Items']
        flag = response.has_key('LastEvaluatedKey')
        count += response['Count']

    response = {'Items': items, 'time': int(time.time()), 'Count': count}
    response_json = json.dumps(response, sort_keys=True, separators=(',', ': '))
    file = open('/tmp/data.json', 'w')
    file.write(response_json)
    file.close()

    s3 = boto3.resource('s3')
    s3.Bucket(WEBSITE_BUCKET).upload_file('/tmp/data.json', 'data/servicesStatus.json')

def lambda_handler(event, context):
    update_json_file_to_s3()

    return "Done."


'''
Script: Image to sketch.
Author@ mujeebishaque
'''

import argparse
import cv2, os

class SketchImage():

    @staticmethod
    def sketch(image_path, destination):
        if not image_path or len(image_path) < 6:
            raise Exception("Check the image path provided.")
        if not destination or len(destination) < 2:
            print("\nINFO: No destination directory provided. Sketch would be saved in the current directory.")

        filename = str(image_path).split(os.sep)[-1]
        _filename, tail = filename.split('.')

        _image = cv2.imread(image_path, cv2.IMREAD_GRAYSCALE)

        invert = cv2.bitwise_not(_image)  # helps in masking of the image
        blur = cv2.GaussianBlur(invert, (21, 21), 0)
        invertedblur = cv2.bitwise_not(blur)
        sketch = cv2.divide(_image, invertedblur, scale=256.0)

        try:
            cv2.imwrite(_filename+'_gray.'+tail, sketch)  # converted image is saved as mentioned name
        except Exception:
            raise Exception("Can't write data to the folder.")

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='''
    This script takes in image path and converts the image into sketch and
    outputs to it either in the provided directory or in the current directory where this script is being executed.''', epilog="pythonvoyage.com")

    parser.add_argument('-i', '--image', required=True, type=str, help='Image path input like C:/windows/Dell/Desktop/demo.png')
    parser.add_argument('-o', '--output', required=False, type=str, help='Output path like C:/windows/Dell/Desktop/Output/image.png')

    argument = vars(parser.parse_args())

    image_path  = argument['image']
    output_path = argument['output']
    try:
        SketchImage.sketch(image_path, output_path)
    except IOError:
        raise Exception("Can't sketchify provided image.")


import json
import glob
import os
from dashify.visualization.data_model.grid_search_result import GridSearchResult
from dashify.visualization.data_model.experiment import Experiment
from typing import Dict, List

class LocalDataLoader:
    """
    Static class that loads the experiments of a given grid search from disk
    and creates a `GridSearchResult` object.
    """

    @staticmethod
    def get_grid_search_results(gs_log_dir: str) -> GridSearchResult:
        """
        Creates a `GridSearchResult` from a given logging directory.
        :param gs_log_dir: Path to grid search logs
        :return: GridSearchResult
        """
        gs_result = GridSearchResult(gs_log_dir)
        config_paths = glob.glob(os.path.join(gs_log_dir, "**/config.json"), recursive=True)
        metric_paths = glob.glob(os.path.join(gs_log_dir, "**/metrics.json"), recursive=True)

        LocalDataLoader._check_integrity_of_logs(config_paths, metric_paths)

        # load configs and metrics into dictionaries (experiment_id -> loaded_resource)
        configs = {LocalDataLoader._resource_path_to_experiment_id(config_path, gs_log_dir): LocalDataLoader._load_file(resource_path=config_path)
                   for config_path in config_paths}
        metrics = {LocalDataLoader._resource_path_to_experiment_id(metric_path, gs_log_dir): LocalDataLoader._load_file(resource_path=metric_path)
                   for metric_path in metric_paths}

        # create experiment and store in grid search result object
        for experiment_id in configs.keys():
            experiment = Experiment(config=configs[experiment_id],
                                    metrics=metrics[experiment_id],
                                    identifier=experiment_id)
            gs_result.add_experiment(experiment)
        return gs_result

    @staticmethod
    def _check_integrity_of_logs(config_paths: List[str], metric_paths: List[str]):
        """
        Makes sure that each experiment has all files present.
        :param config_paths: List of config file paths
        :param metric_paths: List of metric file paths
        :return:
        """
        experiments_configs = [os.path.dirname(c) for c in config_paths]
        experiments_metrics = [os.path.dirname(m) for m in metric_paths]
        intersection = [e for e in experiments_metrics if e in experiments_configs]
        if len(intersection) != len(experiments_configs) or len(intersection) != len(experiments_metrics):
            raise Exception("Dataset corrupt!!!")

    @staticmethod
    def _load_file(resource_path: str):
        with open(resource_path, "r") as f:
            return json.load(f)

    @staticmethod
    def _resource_path_to_experiment_id(experiment_file_path: str, gs_log_dir: str) -> str:
        """
        Maps the file path of a config or metrics file to the respective experiment id.
        :param experiment_file_path:
        :param gs_log_dir:
        :return:
        """
        return os.path.dirname(os.path.relpath(experiment_file_path, gs_log_dir))


from model.detection import detection
from model.detection.lossFuntion import loss_region
import cv2
import os
import tensorflow as tf
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint
import tensorflow.keras.backend as K
import numpy as np

enco_path = "data/image/resized"
original_file_list = os.listdir(enco_path)
num_enco_files = len(original_file_list)

deco_path = "data/image/hitmap"
heatmap_file_list = os.listdir(deco_path)
num_deco_f = len(heatmap_file_list)

X = None
Y = None

for i in range(num_enco_files):
    img_path = enco_path + "/after_" + str(i) + ".jpg"
    print(img_path)
    img = cv2.imread(img_path)
    img = tf.image.convert_image_dtype(img, tf.float32)
    img = np.reshape(img, ((1,) + img.shape))  # 차원추가
    if i == 0:
        X = img
    else:
        X = np.concatenate((X, img), axis=0)  # 추가된 차원(4차원) 방향으로 이미지 연결

for i in range(num_deco_f):
    img_path = deco_path + "/heatmap_" + str(i) + ".jpg"
    print(img_path)
    img = cv2.imread(img_path)
    img = tf.image.convert_image_dtype(img, tf.float32)
    img = np.reshape(img, ((1,) + img.shape))  # 차원추가
    if i == 0:
        Y = img
    else:
        Y = np.concatenate((Y, img), axis=0)  # 추가된 차원(4차원) 방향으로 이미지 연결

print("model 생성시작...")
# model
model = detection.Detection_model()
model.summary()
print("model 생성완료")

# callback
callbacks_list = [
    EarlyStopping(
        monitor="val_acc",
        patience=1,
    ),
    ModelCheckpoint(
        filepath="model.hr",  # 모델파일 경로,
        monitor="val_loss",
        save_best_only=True,
    ),
]

# compile
model.compile(optimizer="adam", loss=loss_region, metrics=["accuracy"])  # acc

print("model 학습 시작...")

# fit
model.fit(X, Y, batch_size=1, epochs=1, verbose=True, callbacks=callbacks_list)

print("model 학습 종료")


import requests
import json

BASE_URL = 'localhost:9091/'

print("\n")
#일반 get request
response = requests.get(BASE_URL + 'userList/v1.0')
data = response.json()
print("1. 유저리스트 조회 : ", data['check'])

#일반 post request에 params 세팅 후 전송
params = {'key1' : 'value1',
          'key2' : 'value2'}
response = requests.post(BASE_URL + 'userInfo/v1.0/post', params)
data = response.json()
print("2. 유저정보 업데이트 is : ", data['check'])

#PUT 헤더와 파라미터 세팅
header = {
    "accept" : "application/json",
    "Content-Type" : "application/json;charset=UTF-8"
}
params = {'key1' : 'value1',
          'key2' : 'value2'}
response = requests.put(BASE_URL + 'userInfo/v1.0/put', headers = header, data = json.dumps(params))
data = response.json()
print("3. 유저정보 저장 is : ", data['check'])

#GET 요청 파라미터 추가 타입
response = requests.get(BASE_URL + 'userDetail/v1.0/get/' + str(userList['response'][0]['userSeq']))
data = response.json()
print("4. 유저상세조회 is : ", data['check'])

#GET 요청 파라미터 세팅 타입
params = {'key1' : 'value1',
          'key2' : 'value2'}
response = requests.get(BASE_URL + 'stock/v1.0/list', params=params)
data = response.json()
print("5. 유저특성조회 is : ", data['check'])


import torch
from torch import nn
from d2l import torch as d2l

class BatchNorm(nn.Module):
    def __init__(self, num_features, num_dims):
        """
        num_features: nums of outputs for fc
                      nums of output channels for convolution
        num_dims: 2 for fc,
                  4 for conv
        """
        super().__init__()
        if num_dims == 2:
            shape = (1, num_features)
        else:
            shape = (1, num_features, 1, 1)
        # 参数赋形初始化
        self.gamma = nn.Parameter(torch.ones(shape))
        self.beta = nn.Parameter(torch.zeros(shape))
        self.moving_mean = torch.zeros(shape)
        self.moving_var = torch.zeros(shape)

    def batch_norm(X, gamma, beta, moving_mean, moving_var, eps, momentum):
        # 推理模式
        if not torch.is_grad_enabled():
            X_hat = (X - moving_mean) / torch.sqrt(moving_var + eps)
        # 训练模式
        else:
            assert len(X.shape) in (2, 4)
            # 当X为全连接层上的小批量，对小批量N的一维样本求均值和方差
            if len(X.shape) == 2:
                mean = X.mean(dim=0, keepdim=True) # shape = (1, X.shape[1])
                var = ((X - mean)**2).mean(dim=0, keepdim=True)
            # 当X为卷积层上的小批量，对小批量N的每一个通道求HxW的均值和方差
            else:
                mean = X.mean(dim=(0,2,3), keepdim=True) # shape = (1, C, 1, 1)
                var = ((X - mean)**2).mean(dim=(0,2,3), keepdim=True)
            # 归一化
            X_hat = (X - mean) / torch.sqrt(var + eps)
            # 持续更新所有样本的均值和方差，为推理阶段的BN准备
            moving_mean = momentum * moving_mean + (1.0 - momentum) * mean
            moving_var = momentum * moving_var + (1.0 - momentum) * var
        # 可训练的伸缩参数和偏移参数，输出最终的BN结果
        Y = gamma * X_hat + beta
        return Y, moving_mean, moving_var

    def forward(self, X):
        if self.moving_mean.device != X.device:
            self.moving_mean = self.moving_mean.to(X.device)
            self.moving_var = self.moving_var.to(X.device)

        Y, self.moving_mean, self.moving_var = batch_norm(X, self.gamma, self.beta,
                                                          self.moving_mean, self.moving_var,
                                                          eps=1e-5, momentum=0.9)
        return Y

# 应用`BatchNorm`于LeNet模型
net = nn.Sequential(nn.Conv2d(1, 6, kernel_size=5),
                    BatchNorm(num_features=6, num_dims=4),
                    nn.Sigmoid(),
                    nn.MaxPool2d(kernel_size=2, stride=2),
                    nn.Conv2d(6, 16, kernel_size=5),
                    BatchNorm(num_features=16, num_dims=4),
                    nn.Sigmoid(),
                    nn.MaxPool2d(kernel_size=2, stride=2),
                    nn.Flatten(),
                    nn.Linear(16 * 4 * 4, 120),
                    BatchNorm(num_features=120, num_dims=2),
                    nn.Sigmoid(),
                    nn.Linear(120, 84),
                    BatchNorm(num_features=84, num_dims=2),
                    nn.Sigmoid(),
                    nn.Linear(84, 10))

# 在Fashion-MNIST数据集上训练网络
lr, num_epochs, batch_size = 1.0, 10, 256
train_iter, test_iter = d2l.load_data_fashion_mnist(batch_size)
d2l.train_ch6(net, train_iter, test_iter, num_epochs, lr, d2l.try_gpu())


import random

import pygame

SCREEN_SIZE = [1000, 800]
WHITE = (255, 255, 255)

CELL_RADIUS = 20
SNAKE_COLOR = (0, 0, 0)

FOOD_COLOR = (100, 100, 100)
FOOD_RADIUS = 20

UPDATE = pygame.USEREVENT + 1
Food = pygame.USEREVENT + 2

LIGHT_GREY = (100, 100, 100)
MSG_POSITION = (300, 500)

def init_game():
    pygame.init()
    pygame.display.set_caption("snake 贪吃蛇")
    pygame.time.set_timer(UPDATE, 500)
    pygame.time.set_timer(Food, 3000)

class Game:
    def __init__(self):
        self.running = True
        self.screen = pygame.display.set_mode(SCREEN_SIZE)
        self.snake = Snake()
        self.food = None
        self.message = None

class Cell:
    def __init__(self, x, y):
        self.x = x
        self.y = y

    def to_tuple(self):
        return self.x, self.y

    def copy(self):
        return Cell(self.x, self.y)

    def update(self, direction):
        if direction == "U":
            self.y -= CELL_RADIUS * 2
        elif direction == "D":
            self.y += CELL_RADIUS * 2
        elif direction == "L":
            self.x -= CELL_RADIUS * 2
        elif direction == "R":
            self.x += CELL_RADIUS * 2

class Snake:
    def __init__(self):
        cell_diameter = CELL_RADIUS * 2
        x = (SCREEN_SIZE[0] / cell_diameter // 2) * cell_diameter
        y = (SCREEN_SIZE[1] / cell_diameter // 2) * cell_diameter

        self.body = [Cell(x, y)]
        self.cell_size = CELL_RADIUS
        self.color = SNAKE_COLOR
        self.direction = "L"

    def update(self):
        head = self.body[0].copy()
        self.body.pop()
        head.update(self.direction)
        self.body.insert(0, head)

def is_snake_food_collide():
    head = game.snake.body[0].copy()
    head.update(game.snake.direction)
    return (game.food[0] - head.x) == 0 and (game.food[1] - head.y) == 0

def update():
    check_snake_dir()
    check_food()
    check_head_body_collision()
    check_out_boundary()
    check_win()

    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            game.running = False
        if event.type == UPDATE:
            game.snake.update()
        if event.type == Food:
            generate_food()

def check_win():
    length = len(game.snake.body)
    cell_diameter = CELL_RADIUS * 2
    max_cell_x = SCREEN_SIZE[0] // cell_diameter
    max_cell_y = SCREEN_SIZE[1] // cell_diameter
    if length == 15:  # max_cell_x * max_cell_y:
        game.running = False
        game.message = "You WIN the game. Should restart? y/n"

def check_out_boundary():
    cell_diameter = 2 * CELL_RADIUS
    out_boundary_x_cells = [cell for cell in game.snake.body if
                            cell.x < cell_diameter or cell.x > SCREEN_SIZE[0] - cell_diameter]
    out_boundary_y_cells = [cell for cell in game.snake.body if
                            cell.y < cell_diameter or cell.y > SCREEN_SIZE[1] - cell_diameter]
    if len(out_boundary_x_cells) > 0 or len(out_boundary_y_cells) > 0:
        game.running = False
        game.message = "Snake is out of boundary. Should restart? y/n"

def check_head_body_collision():
    head = game.snake.body[0]
    body = game.snake.body
    try:
        index = [cell.to_tuple() for cell in body[1:]].index(head.to_tuple())
    except ValueError:
        index = -1
    if len(game.snake.body) > 1 and index > -1:
        game.snake.body = game.snake.body[:index]

def check_food():
    if game.food is not None and is_snake_food_collide():
        cell = Cell(game.food[0], game.food[1])
        game.snake.body.insert(0, cell)
        game.food = None

def check_snake_dir():
    pressed_keys = pygame.key.get_pressed()
    if pressed_keys[pygame.K_UP] and not game.snake.direction == "D":
        game.snake.direction = "U"
    if pressed_keys[pygame.K_DOWN] and not game.snake.direction == "U":
        game.snake.direction = "D"
    if pressed_keys[pygame.K_LEFT] and not game.snake.direction == "R":
        game.snake.direction = "L"
    if pressed_keys[pygame.K_RIGHT] and not game.snake.direction == "L":
        game.snake.direction = "R"

def generate_food():
    while game.food is None:
        cell_diameter = CELL_RADIUS * 2
        rand_x = random.randint(1, (SCREEN_SIZE[0] / cell_diameter) - 1) * cell_diameter
        rand_y = random.randint(1, (SCREEN_SIZE[1] / cell_diameter) - 1) * cell_diameter
        if (rand_x, rand_y) not in [cell.to_tuple() for cell in game.snake.body]:
            game.food = (rand_x, rand_y)

def draw():
    game.screen.fill(WHITE)
    if game.running:
        draw_snake()
        draw_food()
    else:
        draw_restart()
    pygame.display.flip()

def draw_restart():
    font = pygame.font.Font(None, 30)
    restart_msg = font.render(game.message, True, LIGHT_GREY)
    game.screen.blit(restart_msg, MSG_POSITION)

def draw_food():
    if game.food is not None:
        pygame.draw.circle(game.screen, FOOD_COLOR, game.food, FOOD_RADIUS)

def draw_snake():
    for cell in game.snake.body:
        pygame.draw.circle(game.screen, game.snake.color, cell.to_tuple(), game.snake.cell_size)

def check_reatart():
    while not game.running:
        for event in pygame.event.get():
            if event.type == pygame.KEYUP and event.key == pygame.K_y:
                game.snake = Snake()
                game.running = True
            if event.type == pygame.KEYUP and event.key == pygame.K_n:
                return

if __name__ == "__main__":
    init_game()
    game = Game()
    while game.running:
        update()
        draw()
        check_reatart()

    pygame.quit()


import numpy as np
import matplotlib.pyplot as plt
from data.pca import perform_pca

def read_data(file_path):
    '''
    read file path and return the dataset
    input(s):
      file_path (string): path of txt file
    output(s):
      dataset (ndarray): the dataset
    '''
    dataset = []
    # Open the file in read mode
    file = open(file_path, 'r')
    for f in file:
        # Split the content using commas
        split_content = f.split(',')
        split_content[-1] = split_content[-1][0]
        dataset.append(split_content)
    dataset = np.array(dataset).astype('float64')
    return dataset

def plot_features(dataset , label_names=[]):
    length = len(label_names)
    labels_column = dataset[:,-1]
    number_of_features = dataset.shape[1]
    labels = np.unique(labels_column).astype('int')
    if length != 0 and length !=len(labels):
        raise ValueError('the number of label names and features should be the same')
    if length == 0: label_names = labels
    for n in range(number_of_features-1):
        for l in labels:
            condition = np.where(labels_column == l)
            plt.hist(dataset[:, n][condition], density = True, bins=50, alpha=0.5, label=label_names[l], edgecolor='gray')
        plt.title('featre number' + str(n+1), alpha=0.7)
        plt.legend()
        plt.savefig('results/feature figures/'+ 'featre number' + str(n+1))
        plt.show()

def LDA(dataset, label_names=[]):
    X = dataset[:,:-1]
    y = dataset[:,-1]
    length = len(label_names)
    labels = np.unique(y).astype('int')
    if length != 0 and length !=len(labels):
        raise ValueError('the number of label names and features should be the same')
    if length == 0: label_names = labels
    condition1 = np.where(y == 0)
    condition2 = np.where(y == 1)
    data1 = X[condition1]
    data2 = X[condition2]
    # Compute class means
    mean_class1 = np.mean(data1, axis=0)
    mean_class2 = np.mean(data2, axis=0)
    # Compute the within-class scatter matrix
    scatter_within = np.dot((data1 - mean_class1).T, (data1 - mean_class1)) + \
                    np.dot((data2 - mean_class2).T, (data2 - mean_class2))
    # Compute the between-class scatter matrix
    scatter_between = np.outer(mean_class1 - mean_class2, mean_class1 - mean_class2)
    # Solve the generalized eigenvalue problem
    eigenvalues, eigenvectors = np.linalg.eig(np.linalg.inv(scatter_within).dot(scatter_between))
    # Sort eigenvectors by eigenvalues
    sorted_indices = np.argsort(eigenvalues)[::-1]
    eigenvalues = eigenvalues[sorted_indices]
    eigenvectors = eigenvectors[:, sorted_indices]
    # Choose the top k eigenvectors (k is the desired dimensionality)
    k = 1
    transformation_matrix = eigenvectors[:, :k]
    # Project the data onto the new subspace
    lda_data = np.dot(X, transformation_matrix)
    # Plot the LDA-transformed data
    plt.hist(lda_data[y == 0], bins=50, alpha=0.5, label=label_names[0], edgecolor='gray')
    plt.hist(lda_data[y == 1], bins=50, alpha=0.5, label=label_names[1], edgecolor='gray')
    plt.xlabel('LDA Component')
    plt.ylabel('Frequency')
    plt.legend()
    plt.title('Histogram of LDA-Transformed Data')
    plt.savefig('results/LDA figure/'+ 'LDA')
    plt.show()

def plot_pca2(dataset, label_names=[]):
    X = dataset[:,:-1]
    y = dataset[:,-1]
    length = len(label_names)
    labels = np.unique(y).astype('int')
    if length != 0 and length !=len(labels):
        raise ValueError('the number of label names and features should be the same')
    if length == 0: label_names = labels
    k = 2
    pca_data = perform_pca(X, k)
    # Plot the PCA-transformed data
    condition1 = np.where(y == 0)
    condition2 = np.where(y == 1)
    data1 = pca_data[condition1]
    data2 = pca_data[condition2]
    plt.scatter(data1[:,0], data1[:,1], alpha=0.7, label=label_names[0])
    plt.scatter(data2[:,0], data2[:,1], alpha=0.7, label=label_names[1])
    plt.xlabel('PCA Component')
    plt.ylabel('y')
    plt.title('PCA-Transformed Data')
    plt.savefig('results/pca2 figure/'+ 'pca2')
    plt.show()

def correlation_heatmap(hitmap_data, labels=None, title="Correlation Heatmap", cmap="gray_r", name='correlation heatmap'):
    correlation_matrix = np.corrcoef(hitmap_data, rowvar=False)
    plt.figure(figsize=(10, 8))
    plt.imshow(correlation_matrix, cmap=cmap)
    plt.colorbar()
    plt.xticks(np.arange(len(labels)), labels)
    plt.yticks(np.arange(len(labels)), labels)
    plt.title(title)
    plt.savefig('results/correlation heatmaps/'+ name)
    plt.show()

def cross_features(dataset, n_feature1, n_feature2, label_names=[]):
    X = dataset[:,:-1]
    y = dataset[:,-1]
    length = len(label_names)
    labels = np.unique(y).astype('int')
    if length != 0 and length !=len(labels):
        raise ValueError('the number of label names and features should be the same')
    if length == 0: label_names = labels
    # Plot the cross features
    condition1 = np.where(y == 0)
    condition2 = np.where(y == 1)
    data1 = X[condition1]
    data2 = X[condition2]
    plt.scatter(data1[:,n_feature1], data1[:,n_feature2], alpha=0.7, label=label_names[0])
    plt.scatter(data2[:,n_feature1], data2[:,n_feature2], alpha=0.7, label=label_names[1])
    plt.xlabel('feature ' + str(n_feature1))
    plt.ylabel('feature ' + str(n_feature2))
    plt.title('cross features')
    plt.savefig('results/cross features/'+ str(n_feature1) + '_and_' + str(n_feature2))
    plt.show()


import numpy as np
from neuron import h,gui
import signal
import pickle
from matplotlib import pyplot as plt
from glob import glob
import pandas as pd
import sys
from extra_function import load_ASC,SIGSEGV_signal_arises
from read_spine_properties import get_spine_xyz,get_n_spinese, get_spine_part
signal.signal(signal.SIGSEGV, SIGSEGV_signal_arises)

if len(sys.argv) != 7:
    print("sys.argv not running and with length",len(sys.argv))
    cells= ['2017_05_08_A_4-5','2017_05_08_A_5-4','2017_03_04_A_6-7']
    file_type='new.ASC'
    folder_='/ems/elsc-labs/segev-i/moria.fridman/project/analysis_groger_cells'
    with_plot=True
else:
    print("sys.argv is correct and running")
    cells = [sys.argv[1],sys.argv[2],sys.argv[3]]
    file_type=sys.argv[4]
    folder_=sys.argv[5]
    with_plot=eval(sys.argv[6])
folder_data=folder_+'/cells_initial_information/'
folder_save=folder_+'/cells_outputs_data/'

def synaptic_loc(cell_dir,syn_poses_list,with_plot=False, part='all', save_place=''):
    dict2={}
    cell=None
    cell=load_ASC(cell_dir)
    #syn_pose should be (x,y,z) coordinates
    # h.load_file("import3d.hoc")
    h.load_file("nrngui.hoc")
    secs,dends,dists,dends_name,dis_from_soma,sec_name,sec_num,seg_num=[],[],[],[],[],[],[],[]
    for i in range(len(syn_poses_list)):
        secs.append(None)
        dends.append(None)
        sec_num.append(None)
        sec_name.append(None)
        seg_num.append(None)
        dists.append(10000)
        dends_name.append(None)
        dis_from_soma.append(None)

    if part == 'all':
        relevant_sections = cell.all_sec()
    elif part == 'basal':
        relevant_sections = cell.dend
    elif part == 'apical':
        if len(cell.apic)==0:
            relevant_sections = cell.dend
        else:
            relevant_sections = cell.apic
    else:
        raise BaseException('the part is not good '+str(part))
    all_points = []
    for sec in relevant_sections:
        lens = []
        initial_point = np.array([sec.x3d(0), sec.y3d(0), sec.z3d(0)])
        for i in range(sec.n3d()):
            lens.append(np.linalg.norm(initial_point - np.array([sec.x3d(i), sec.y3d(i), sec.z3d(i)])))
            initial_point = np.array([sec.x3d(i), sec.y3d(i), sec.z3d(i)])
        total_len = np.sum(lens)
        accumalate_len = 0
        initial_point = np.array([sec.x3d(0), sec.y3d(0), sec.z3d(0)])
        points = [initial_point]
        for i in range(1,sec.n3d()):
            dend_pos = np.array([sec.x3d(i), sec.y3d(i), sec.z3d(i)])
            points_diffrance = dend_pos-initial_point
            distance = np.linalg.norm(initial_point - dend_pos)
            number_of_steps =int(np.ceil(distance))
            for step_number in range(1, number_of_steps, 1):
                intermideate_point = initial_point.copy() + points_diffrance*step_number/number_of_steps
                points.append(intermideate_point)
            points.append(dend_pos)
            initial_point = dend_pos
        initial_point = points[0]
        all_points+=points
        for point in points[1:]:
            dend_pos = point
            accumalate_len += np.linalg.norm(initial_point - dend_pos)
            initial_point = dend_pos
            for j, syn_pos in enumerate(syn_poses_list):
                if np.linalg.norm(syn_pos - dend_pos) < dists[j]:
                    dists[j] = np.linalg.norm(syn_pos - dend_pos)
                    secs[j] = [sec, accumalate_len / total_len]
                    dends[j]=[sec,round(accumalate_len / total_len,3)]
                    dends_name[j]=[str(sec)[str(sec).find('>')+2:],round(accumalate_len / total_len,3)]
                    sec_name[j]=str(sec)[str(sec).find('>')+2:]
                    seg_num[j]=round(accumalate_len / total_len,3)
                    sec_num[j]=int(str(sec)[str(sec).find('[')+1:-1])
    if with_plot:
        plt.figure()
        for p in all_points:
            plt.scatter(p[0], p[1], color='black',s=0.5)
    dend_pos_dict={}
    for j,dend in enumerate(dends_name):
        sec=eval('cell.'+dend[0])
        initial_point = np.array([sec.x3d(0), sec.y3d(0), sec.z3d(0)])
        points_dend = [initial_point]
        for i in range(1,sec.n3d()):
            dend_pos = np.array([sec.x3d(i), sec.y3d(i), sec.z3d(i)])
            # points_diffrance = dend_pos-initial_point
            # distance = np.linalg.norm(initial_point - dend_pos)
            # number_of_steps =int(np.ceil(distance))
            # for step_number in range(1, number_of_steps, 1):
            #     intermideate_point = initial_point.copy() + points_diffrance*step_number/number_of_steps
            #     points_dend.append(intermideate_point)
            points_dend.append(dend_pos)
            dend_pos_dict[j]=points_dend
            # initial_point = dend_pos
            if with_plot:
                plt.scatter(dend_pos[0], dend_pos[1], color='g',s=0.5)
        # for p in points_dend:
        #     plt.scatter(p[0], p[1], color='g',s=0.5)
    if with_plot:
        for j, syn_pos in enumerate(syn_poses_list):
            # if type(syn_pos)!=list:
            #     syn_pos=[syn_pos]
            dis_from_soma[j]=syn_dis_from_soma(cell,dends_name[j])
            plt.scatter(syn_pos[0], syn_pos[1],s=0.7, color='cyan')
            plt.text(-50,-50,str(syn_pos)+'dis from soma='+str(dis_from_soma[j]))
        color_code={'basal':'blue','apical':'red','axon':'green','soma':'purple','synapse':'cyan','syn_trunk':'green'}
        for i in range(cell.soma.n3d()):
            plt.scatter(cell.soma.x3d(i), cell.soma.y3d(i),lw=cell.soma.diam3d(i), color=color_code['soma'])

        legend_elements = [
        plt.scatter(syn_pos[0], syn_pos[1], color='black', s=0.5, label="all_point"),
        plt.scatter(syn_pos[0], syn_pos[1], color='green',s=0.5, label="syn_trunk"),
        plt.scatter(syn_pos[0], syn_pos[1], color=color_code['soma'],s=0.5, label="soma"),
        plt.scatter(syn_pos[0], syn_pos[1], color='cyan', s=0.5, label="synapse")
            ]
        plt.legend(handles=legend_elements, loc="best")
        plt.savefig(save_place+'.pdf')
        plt.savefig(save_place)
        plt.close()
    with open(save_place + '_neuron_morphology.p', 'wb') as f:
        pickle.dump({"all_point":all_points,"synaptic_dend":dend_pos_dict,"syn_pos":xyz,"syn_sec_pos":dends_name}, f)

    dict =  { 'sec_name':sec_name,'sec_num':sec_num,'seg_num':seg_num,'place_name':dends_name,'dist_from_soma':dis_from_soma,'dist':dists, 'part':part}
    print(dict)

    try_save_dict(dict,folder_save+cell_name+'/','synaptic_location'+file_type)
    for i in range(len(syn_poses_list)):
        dict2[str(i)] = {'sec_name':sec_name[i],'sec_num':sec_num[i],'seg_num':seg_num[i],'place_name':dends_name[i],'dist_from_soma':dis_from_soma[i],'dist':dists[i], 'part':part}
        print(dict2)

    try_save_dict(dict2,folder_save+cell_name+'/','synaptic_location_seperate'+file_type)
    return dict,dict2

def syn_dis_from_soma(cell,syn_loc):
    h.distance(0, 0.5, sec=cell.soma)
    synapses_dis_from_soma=h.distance(eval('cell.' + syn_loc[0])(syn_loc[1]))
    return synapses_dis_from_soma
def try_save_dict(dict,folder_save,name):
    try:
        with open(folder_save + name+'.p', 'wb') as f:
            pickle.dump(dict, f)
    except Exception as e:
        print("Error trying to save pickle: " + str(e))
        pass

    with open(folder_save+name+'.txt', 'w') as f:
        try:
            f.write('synaptic_location')
            f.write(dict)
        except Exception as e:
            print("Error trying to save txt: " + str(e))
            pass
    try:
        df1 = pd.DataFrame(dict)
        df1.to_excel(folder_save+name+".xlsx")
    except Exception as e:
        print("Error trying to save xlsx: " + str(e))
        pass
if __name__=='__main__':
    dict2={}
    dict3,dict4={},{}
    name2save=''
    for cell_name in cells:
        xyz,dend_part=[],[]
    # for cell_name in ['2017_05_08_A_5-4']:
        if len(glob(folder_data+cell_name+'/*'+file_type))<1:continue
        dir=glob(folder_data+cell_name+'/*'+file_type)[0]
        for i in range(get_n_spinese(cell_name)):
            print('one syn dict:',dict)
            xyz.append(list(get_spine_xyz(cell_name,i)))
            dend_part.append(get_spine_part(cell_name,i))
        dict1,dict2=synaptic_loc(dir,xyz, part='all', save_place=folder_save+cell_name+'/synapses'+file_type,with_plot=with_plot)
        dict3[cell_name]=dict1
        for key in dict2.keys():
            dict4[cell_name+key]=dict2[key]
        print('more then one syn dict',cell_name,[xyz])
    try_save_dict(dict3,folder_save,'synaptic_location'+file_type)
    try_save_dict(dict4,folder_save,'synaptic_location_seperate'+file_type)


import torch

import resnet_pytorch
import imbalanced_dataset
import numpy as np
import utils
import argparse
from itertools import chain
import os
try:
    from apex import amp
except ImportError:
    amp = None
import train
import initialise_model

def main(args):
    dset_name = args.dset_name
    train_dir = os.path.join(args.data_path, "train")
    val_dir = os.path.join(args.data_path, "val")
    dataset, dataset_test, _, test_sampler = train.load_data(train_dir, val_dir, args)

    num_classes = len(dataset.cls_num_list)

    data_loader_test = torch.utils.data.DataLoader(
        dataset_test, batch_size=args.batch_size,
        sampler=test_sampler, num_workers=args.workers, pin_memory=True)

    model = initialise_model.get_model(args,num_classes)
    criterion = initialise_model.get_criterion(args,dataset)
    checkpoint = torch.load(args.load_from, map_location='cpu')
    model.load_state_dict(checkpoint['model'])

    model.to('cuda')
    if args.apex:
        optimizer = torch.optim.SGD(
            model.parameters(), lr=0.01, momentum=0.0, weight_decay=0.0)
        model, optimizer = amp.initialize(model, optimizer,
                                          opt_level=args.apex_opt_level
                                          )

    avg_acc,preds,targets=evaluate(model.cuda(), data_loader_test, device='cuda',criterion=criterion)

    print(f'Avg Acc is: {avg_acc}')

    f,c,r = shot_acc(np.array(preds),np.array(targets),dataset.targets)

    print(f'Many shot Acc is: {f}, median shot Acc is: {c}, low shot Acc is: {r}')

    return 0

def shot_acc (preds, labels, train_targets, many_shot_thr=100, low_shot_thr=20, acc_per_cls=False):

    if isinstance(train_targets, np.ndarray):
        training_labels = np.array(train_targets).astype(int)
    else:
        training_labels = np.array(train_targets).astype(int)

    if isinstance(preds, torch.Tensor):
        preds = preds.detach().cpu().numpy()
        labels = labels.detach().cpu().numpy()
    elif isinstance(preds, np.ndarray):
        pass
    else:
        raise TypeError('Type ({}) of preds not supported'.format(type(preds)))
    train_class_count = []
    test_class_count = []
    class_correct = []
    for l in np.unique(labels):
        train_class_count.append(len(training_labels[training_labels == l]))
        test_class_count.append(len(labels[labels == l]))
        class_correct.append((preds[labels == l] == labels[labels == l]).sum())

    many_shot = []
    median_shot = []
    low_shot = []
    for i in range(len(train_class_count)):
        if train_class_count[i] > many_shot_thr:
            many_shot.append((class_correct[i] / test_class_count[i]))
        elif train_class_count[i] < low_shot_thr:
            low_shot.append((class_correct[i] / test_class_count[i]))
        else:
            median_shot.append((class_correct[i] / test_class_count[i]))

    if len(many_shot) == 0:
        many_shot.append(0)
    if len(median_shot) == 0:
        median_shot.append(0)
    if len(low_shot) == 0:
        low_shot.append(0)

    if acc_per_cls:
        class_accs = [c / cnt for c, cnt in zip(class_correct, test_class_count)]
        return np.mean(many_shot), np.mean(median_shot), np.mean(low_shot), class_accs
    else:
        return np.mean(many_shot), np.mean(median_shot), np.mean(low_shot)

def evaluate(model, data_loader, device, print_freq=10,criterion=None):
    model.eval()
    metric_logger = utils.MetricLogger(delimiter="  ")
    header = 'Test:'
    predictions=[]
    targets=[]
    with torch.no_grad():
        for image, target in metric_logger.log_every(data_loader, print_freq, header):
            image = image.to(device, non_blocking=True)
            target = target.to(device, non_blocking=True)
            output = model(image)
            if hasattr(criterion, 'iif'):
                output=criterion(output,infer=True)
            acc1, acc5 = utils.accuracy(output, target, topk=(1, 5))

            predictions=list(chain(predictions,output.argmax(axis=1).tolist()))
            targets=list(chain(targets,target.tolist()))

            # FIXME need to take into account that the datasets
            # could have been padded in distributed setup
            batch_size = image.shape[0]
#             metric_logger.update(loss=loss.item())
            metric_logger.meters['acc1'].update(acc1.item(), n=batch_size)
            metric_logger.meters['acc5'].update(acc5.item(), n=batch_size)
    # gather the stats from all processes
    metric_logger.synchronize_between_processes()

    print(' * Acc@1 {top1.global_avg:.3f} Acc@5 {top5.global_avg:.3f}'
          .format(top1=metric_logger.acc1, top5=metric_logger.acc5))

    return metric_logger.acc1.global_avg,predictions,targets

def get_args_parser():
    parser = argparse.ArgumentParser(description='Parse arguments for per shot acc.')
    parser.add_argument('--distributed', default=False)
    parser.add_argument('--dset_name', default='imagenet_lt',type=str, help='Dataset Name imagenet_lt|places')
    parser.add_argument(
        '--data-path', default='../../../datasets/ILSVRC/Data/CLS-LOC/', help='dataset')
    parser.add_argument("--auto-augment", default=None, type=str, help="auto augment policy (default: None)")
    parser.add_argument("--ra-magnitude", default=9, type=int, help="magnitude of auto augment policy")
    parser.add_argument("--augmix-severity", default=3, type=int, help="severity of augmix policy")
    parser.add_argument("--random-erase", default=0.0, type=float, help="random erasing probability (default: 0.0)")
    parser.add_argument('--sampler', default='random', type=str, help='sampling, [random,upsampling,downsampling]')
    parser.add_argument('--iif', default='raw',type=str, help='Type of IIF variant')
    parser.add_argument('--classif', default='ce',type=str, help='Type of classification')
    parser.add_argument('--classif_norm', default=None,type=str, help='Type of classifier Normalisation {None,norm,cosine')
    parser.add_argument('--load_from', default='', help='load wweights only from checkpoint')
    parser.add_argument('-j', '--workers', default=4, type=int, metavar='N',help='number of data loading workers (default: 16)')
    parser.add_argument('-b', '--batch-size', default=256, type=int)
    parser.add_argument('--model', default='resnet50', help='model, E.g.(se_resnext50_32x4d)')
    parser.add_argument('--use_gumbel_se', default=False, help='Gumbel activation in excitation phase of SE',action='store_true')
    parser.add_argument('--use_gumbel_cb', default=False, help='Gumbel activation in spatial gate phase of CB',action='store_true')

    parser.add_argument('--criterion', default='ce',type=str, help='Criterion used for classifier {ce,bce,gce')

    parser.add_argument('--apex', action='store_true',
                        help='Use apex for mixed precision training')
    parser.add_argument(
        "--interpolation", default="bilinear", type=str, help="the interpolation method (default: bilinear)"
    )
    parser.add_argument(
        "--val-resize-size", default=256, type=int, help="the resize size used for validation (default: 256)"
    )
    parser.add_argument(
        "--val-crop-size", default=224, type=int, help="the central crop size used for validation (default: 224)"
    )
    parser.add_argument(
        "--train-crop-size", default=224, type=int, help="the random crop size used for training (default: 224)"
    )
    parser.add_argument(
        "--label-smoothing", default=0.0, type=float, help="label smoothing (default: 0.0)", dest="label_smoothing"
    )
    parser.add_argument('--apex-opt-level', default='O2', type=str,
                        help='For apex mixed precision training'
                             'O0 for FP32 training, O1 for mixed precision training.'
                             'For further detail, see https://github.com/NVIDIA/apex/tree/master/examples/imagenet'
                        )
    parser.add_argument(
        "--cache-dataset",
        dest="cache_dataset",
        help="Cache the datasets for quicker initialization. It also serializes the transforms",
        action="store_true",
    )
    parser.add_argument(
        "--deffered",
        help="Use deferred schedule",
        action="store_true",
    )
    parser.add_argument(
        "--pretrained",
        dest="pretrained",
        help="Use pre-trained models from the modelzoo or local --> [pytorch, path-to-model]",
        default=None, type=str,
    )
    parser.add_argument("--weights", default=None, type=str, help="Dummy input, do not change")
    parser.add_argument('--reduction', default='mean', type=str, help='reduce mini batch')
    parser.add_argument(
        "--test-only",
        dest="test_only",
        help="Dummy input, do not change",
        action="store_true",
    )

    return parser

if __name__ == "__main__":
    args = get_args_parser().parse_args()
    main(args)
    print('end of program')


from django.urls import path
from . import views

urlpatterns = [
    path('v1/',views.view1),
    path('add/',views.add),
    path('hm/',views.home,name='home' ),
    path('ab/',views.about,name='about'),
    path('co/',views.contact,name='contact'),
    path('as/',views.allStudent),
    path('em/',views.allEmployee),

]


'''
Created on Aug 8, 2016
Processing datasets.

@author: Xiangnan He (xiangnanhe@gmail.com)

Modified  on Nov 10, 2017, by Lianhai Miao
'''

import scipy.sparse as sp
import numpy as np
import torch
from torch.utils.data import TensorDataset, DataLoader
from collections import defaultdict
import math

class Data_loader:
    def __init__(self,dataset,batch_size):
        self.dataset = dataset
        self.batch_size = batch_size
        self.datasize = dataset.shape[0]

    def get_batch(self):
        index = torch.randint(self.datasize,[self.batch_size])
        return self.dataset[index]

class GDataset(object):

    def __init__(self, dataname, user_path, group_path, user_in_group_path, num_negatives, test_pos_neg_num):
        '''
        Constructor
        '''
        self.dataname = dataname
        self.user_in_group_path = user_in_group_path
        self.num_negatives = num_negatives
        self.test_pos_neg_num = test_pos_neg_num
        # user data
        self.user_trainMatrix, self.user_explicit_trainMatrix = self.load_rating_file_as_matrix(user_path + "Train.txt")
        self.num_users, self.num_items = self.user_trainMatrix.shape

        self.usertest_input, self.item_test_input = self.get_test_instances(user_path + "Negative.txt")

        # self.u_items_dict, self.i_users_dict = self.get_ui_dict(user_path + "Train.txt")
        # group data
        self.group_trainMatrix, self.gro_explicit_trainMatrix = self.load_rating_file_as_matrix(group_path + "Train.txt")

        self.grouptest_input, self.groupitem_test = self.get_test_instances(group_path + "Negative.txt")
        # self.gro_items_dict, self.i_groups_dict = self.get_ui_dict(group_path + "Train.txt")
        self.gro_members_dict, self.user_groups_dict = self.get_group_member_dict(user_in_group_path)
        self.num_groups = len(self.gro_members_dict)

        self.gro_user = self.get_group_member_input(user_in_group_path)
        self.userTrain = self.get_ug_train_instances(self.user_trainMatrix)
        self.groTrain = self.get_ug_train_instances(self.group_trainMatrix)

    def get_group_member_input(self, user_in_group_path):
        gro_input = []
        member_input = []
        with open(user_in_group_path, 'r') as f:
            line = f.readline().strip()
            while line != None and line != "":
                a = line.split(' ')
                g = int(a[0])
                # g_m_d[g] = []
                for m in a[1].split(','):
                    gro_input.append(g)
                    member_input.append(int(m))
                line = f.readline().strip()
        return gro_input, member_input

    def get_ug_train_instances(self, train):
        user_input, pos_item_input, neg_item_input = [], [], []
        num_users = train.shape[0]
        num_items = train.shape[1]
        for (u, i) in train.keys():
            # positive instance
            for _ in range(self.num_negatives):
                pos_item_input.append(i)
            # negative instances
            for _ in range(self.num_negatives):
                j = np.random.randint(num_items)
                while (u, j) in train:
                    j = np.random.randint(num_items)
                user_input.append(u)
                neg_item_input.append(j)
        u_pi_ni = [[u, pi, ni] for u, pi, ni in zip(user_input, pos_item_input, neg_item_input)]
        return np.array(u_pi_ni)

    def get_group_member_dict(self, user_in_group_path):
        g_m_d = defaultdict(list)
        u_g_d = defaultdict(list)
        with open(user_in_group_path, 'r') as f:
            line = f.readline().strip()
            while line != None and line != "":
                if self.dataname == 'MaFengWo' or self.dataname == 'CAMRa2011':
                    a = line.split(' ')
                    g = int(a[0])
                    # g_m_d[g] = []
                    for m in a[1].split(','):
                        g_m_d[g].append(int(m))
                        u_g_d[int(m)].append(g)
                elif self.dataname == 'ml-latest-small':
                    a = line.split(' ')
                    g = int(a[0])
                    # g_m_d[g] = []
                    for m in a[1:]:
                        g_m_d[g].append(int(m))
                        u_g_d[int(m)].append(g)
                line = f.readline().strip()
        return g_m_d, u_g_d

    # def get_user_groups_dict(self, user_in_group_path):
    #     u_g_d = defaultdict(list)
    #     with open(user_in_group_path, 'r') as f:
    #         line = f.readline().strip()
    #         while line != None and line != "":
    #             a = line.split(' ')
    #             g = int(a[0])
    #             for m in a[1].split(','):
    #                 u_g_d[int(m)].append(g)
    #             line = f.readline().strip()
    #     return u_g_d

    def load_rating_file_as_list(self, filename):
        ratingList = []
        with open(filename, "r") as f:
            line = f.readline()
            while line != None and line != "":
                arr = line.split(" ")
                user, item = int(arr[0]), int(arr[1])
                ratingList.append([user, item])
                line = f.readline()
        return ratingList

    def get_ui_dict(self, filename):
        # user_i -> items: 1: [10, 23]
        with open(filename, 'r') as reader:
            u_i_dict = defaultdict(list)
            i_u_dict = defaultdict(list)
            for line in reader:
                # 162540,32,4,...
                user_id, item_id = map(int, line.split(' ')[:2]) # ',' or '\t'
                u_i_dict[user_id].append(item_id)
                i_u_dict[item_id].append(user_id)

        return u_i_dict, i_u_dict

    def load_negative_file(self, filename):
        testRatingList, negativeList = [], []
        with open(filename, "r") as f:
            line = f.readline()
            while line != None and line != "":
                if line == "\n":
                    line = f.readline()
                    continue
                arr = line.split(" ")
                user, item = eval(arr[0])[0], eval(arr[0])[1]
                testRatingList.append([user, item])
                negatives = []
                for x in arr[1:]:
                    negatives.append(int(x))
                negativeList.append(negatives)
                line = f.readline()
        return testRatingList, negativeList

    def load_rating_file_as_matrix(self, filename):
        # Get number of users and items
        num_users, num_items = 0, 0
        with open(filename, "r") as f:
            line = f.readline()
            while line != None and line != "":
                arr = line.split(" ")
                u, i = int(arr[0]), int(arr[1])
                num_users = max(num_users, u)
                num_items = max(num_items, i)
                line = f.readline()
        # Construct matrix
        mat = sp.dok_matrix((num_users + 1, num_items + 1), dtype=np.float32)
        mat_explicit = sp.dok_matrix((num_users + 1, num_items + 1), dtype=np.float32)
        with open(filename, "r") as f:
            line = f.readline()
            while line != None and line != "":
                arr = line.split(" ")
                if len(arr) > 2:
                    user, item, rating = int(arr[0]), int(arr[1]), eval(arr[2])
                    if (rating > 0):
                        mat[user, item] = 1.0
                        mat_explicit[user, item] = rating
                else:
                    user, item = int(arr[0]), int(arr[1])
                    mat[user, item] = 1.0
                line = f.readline()
        return mat, mat_explicit

    def get_explicit_train_instances(self, train):
        user_input, item_input, rating = [], [], []
        num_users = train.shape[0]
        num_items = train.shape[1]
        max_rating = max(train.values())
        for (u, i) in train.keys():
            # positive instance
            user_input.append(u)
            item_input.append(i)
            rating.append(train[u, i]/max_rating)
            for _ in range(self.num_negatives):
                # negative instances
                j = np.random.randint(num_items)
                while (u, j) in train:
                    j = np.random.randint(num_items)
                user_input.append(u)
                item_input.append(j)
                rating.append(0)
        # pi_ni = [[pi, ni] for pi, ni in zip(pos_item_input, neg_item_input)]
        return user_input, item_input, rating

    def get_train_instances(self, train):
        user_input, pos_item_input, neg_item_input = [], [], []
        num_users = train.shape[0]
        num_items = train.shape[1]
        for (u, i) in train.keys():
            # positive instance
            for _ in range(self.num_negatives):
                pos_item_input.append(i)
            # negative instances
            for _ in range(self.num_negatives):
                j = np.random.randint(num_items)
                while (u, j) in train:
                    j = np.random.randint(num_items)
                user_input.append(u)
                neg_item_input.append(j)
        pi_ni = [[pi, ni] for pi, ni in zip(pos_item_input, neg_item_input)]
        return user_input, pi_ni

    def get_test_instances(self, filename):
        user_input, item_input = [], []
        with open(filename, "r") as f:
            line = f.readline()
            while line != None and line != "":
                if line == "\n":
                    line = f.readline()
                    continue
                if self.dataname == 'MaFengWo' or self.dataname == 'CAMRa2011':
                    arr = line.split(" ")
                    user, pos_item = eval(arr[0])[0], eval(arr[0])[1]
                    user_input.append(user)
                    item_input.append(pos_item)
                    for x in arr[1:]:
                        user_input.append(user)
                        item_input.append(int(x))
                elif self.dataname == 'ml-latest-small':
                    arr = line.split(' ')
                    user = int(arr[0])
                    pos_item = int(arr[1])
                    user_input.append(user)
                    item_input.append(pos_item)
                    for x in arr[2:]:
                        user_input.append(user)
                        item_input.append(int(x))
                line = f.readline()
        return user_input, item_input

    def get_user_dataloader(self, batch_size):
        user, positem_negitem_at_u = self.get_train_instances(self.user_trainMatrix)
        train_data = TensorDataset(torch.LongTensor(user), torch.LongTensor(positem_negitem_at_u))
        user_train_loader = DataLoader(train_data, batch_size=batch_size, shuffle=True)
        return user_train_loader

    def get_group_dataloader(self, batch_size):
        # group and positem_negitem_at_g are two lists
        group, positem_negitem_at_g = self.get_train_instances(self.group_trainMatrix)
        train_data = TensorDataset(torch.LongTensor(group), torch.LongTensor(positem_negitem_at_g))
        group_train_loader = DataLoader(train_data, batch_size=batch_size, shuffle=True, drop_last=True)

        return group_train_loader

    def get_ug_dataloader(self, batch_size):
        gro_input, member_input = self.get_group_member_input(self.user_in_group_path)
        train_data = TensorDataset(torch.LongTensor(gro_input), torch.LongTensor(member_input))
        train_loader = DataLoader(train_data, batch_size=batch_size, shuffle=True, drop_last=True)

        return train_loader

    def get_user_explicit_dataloader(self, batch_size):
        user_input, item_input, rating = self.get_explicit_train_instances(self.user_explicit_trainMatrix)
        train_data = TensorDataset(torch.LongTensor(user_input), torch.LongTensor(item_input), torch.Tensor(rating))
        user_explicit_train_loader = DataLoader(train_data, batch_size=batch_size, shuffle=True, drop_last=True)

        return user_explicit_train_loader

    def get_gro_explicit_dataloader(self, batch_size):
        gro_input, item_input, rating = self.get_explicit_train_instances(self.gro_explicit_trainMatrix)
        train_data = TensorDataset(torch.LongTensor(gro_input), torch.LongTensor(item_input), torch.Tensor(rating))
        gro_explicit_train_loader = DataLoader(train_data, batch_size=batch_size, shuffle=True, drop_last=True)

        return gro_explicit_train_loader

    def get_user_test_dataloader(self):
        test_tensor_data = TensorDataset(torch.LongTensor(self.usertest_input), torch.LongTensor(self.item_test_input))
        test_loader = DataLoader(test_tensor_data,batch_size=self.test_pos_neg_num + 1, shuffle=False)
        return test_loader

    def get_gro_test_dataloader(self):
        test_tensor_data = TensorDataset(torch.LongTensor(self.grouptest_input), torch.LongTensor(self.groupitem_test))
        test_loader = DataLoader(test_tensor_data, batch_size=self.test_pos_neg_num + 1, shuffle=False)
        return test_loader


#!/usr/bin/env python2

from rest_framework import serializers

from domain.achievement.models.Action import Action
from custom_fields.ChoicesField import ChoicesField

class ActionSerializer(serializers.ModelSerializer):
    action_verb = ChoicesField(choices=Action.ACTION_VERBS)

    class Meta:
        model = Action
        fields = (
            'id',
            'goal',
            'action_verb',
            'activity',
            'target_metric',
            'target_metric_unit',
            'created_date',
            'updated_date',
        )


import os
import random
import discord

from discord.ext import commands
from dotenv import load_dotenv
from datetime import timedelta, datetime

load_dotenv()
TOKEN = os.getenv('DISCORD_TOKEN')
date = None

bot = commands.Bot(command_prefix='-')

# display a message in console to tell admins that the bot is online
@bot.event
async def on_ready():
    print(f'{bot.user} has connected to Discord!')

# @bot.event
# async def on_message(message):
#     bot.wait_until_ready()
#     x = random.randint(0, 100)
#     if x == 69:
#         await message.send("https://www.change.org/plzsign_1_")
#     await bot.process_commands(message)

# When ever a new member joins, the bot will send them a dm to welcome them
@bot.event
async def on_member_join(member):
    await member.create_dm()
    await member.dm_channel.send(
        f'Hi {member.name}, welcome to the go server !'
    )

# give meeting time
@bot.command(name='settime', help="Sets the meeting time")
@commands.has_any_role('Officer','Teacher', 483412758621323264)
async def setMeetingTime(ctx, *, message):
    global date
    date = message

#next
@bot.command(name='meeting', help="||Tell you next metting time or something")
async def nine_nine(ctx):
    if ctx.author == bot.user:
        return
    print(date)
    await ctx.reply(date)

#bans a user with a reason
# @bot.command(name='test', help="|| test command don't tuch")
# @commands.has_permissions(ban_members = True)
# async def ban(ctx, member : discord.Member, *, reason = None):
#     await member.ban(reason = reason)

# #give role to a member
@bot.command()
async def giverole(ctx, arg: discord.Member):
    await ctx.send(arg)
    knownrole = discord.utils.get(ctx.guild.roles, name="Mod")
    await arg.add_roles(knownrole)

# #spam
# @bot.command(name='spam')
# async def nine_nine(ctx):
#     for i in range (10):
#         await ctx.send("@everyone")
#         i=i+1

bot.run(TOKEN)


try:
    from commands.creation.creation_commands import CreationCommand
    from dna.dna import Dna
    from dna.dna_sequence import DnaSequence
except ImportError:
    print("Need to fix the installation")
    raise
'''
New Command.
command: new <seq> [@<seq_name>]
if name is not provided a default name is set to be: seq.
'''

class NewCommand(CreationCommand):
    __id = 1

    def __init__(self, args):
        name, dna_sequence = self.parse(args)
        self.__name = self.validate_name(name, 'seq')
        self.__dna_sequence = DnaSequence(dna_sequence)
        self.__dna = Dna(dna_sequence, self.__name)

    def execute(self):
        self.print(Dna.get_id(), self.__dna.get_name(), self.__dna_sequence)
        self.__dna.execute()

    @staticmethod
    def get_id():
        return NewCommand.__id

    @staticmethod
    def set_id(id):
        NewCommand.__id += id


import pyodbc

class PhoneBook:

    def __init__(self, dbname, tname):
        self.dbname = dbname
        self.tname = tname
        self.connString = 'Driver={SQL Server};Server=DESKTOP-DH4N9T0\SQLEXPRESS01;Database='+dbname+';Trusted_Connection=yes;'
        conn = pyodbc.connect(self.connString)
        cursor = conn.cursor()
        cursor.execute(f"SELECT * FROM {tname}")
        self.pb = dict()
        for contact in cursor:
            self.pb.update({contact[0]:contact[1]})
        conn.close()

    def listContacts(self):
        print("Name\t\tPhone")
        for i in sorted(self.pb.keys()):
            print("{}\t\t{}".format(i, self.pb[i]))

    def addContact(self, name, phno):
        if name not in self.pb.keys():
            self.pb.update({name:phno})
            print("1 entry added")
        else:
            print("entry exists")

    def searchByName(self, name):
        if name in self.pb.keys():
            print("entry found - ", name, "-", self.pb[name])
        else:
            print("entry not found")

    def searchByNumber(self, phno):
        if phno in self.pb.values():
            for k,v in self.pb.items():
                if phno == v:
                    name = k
                    break
            print("entry found - ", name, "-", phno)
        else:
            print("entry not found")

    def deleteContact(self, name):
        if name in self.pb.keys():
            del self.pb[name]
            print("1 entry ({}) deleted".format(name))
        else:
            print("entry does not exist")

    def commit(self):
        conn = pyodbc.connect(self.connString)
        cursor = conn.cursor()
        cursor.execute(f"TRUNCATE TABLE {self.tname}")
        for k,v in self.pb.items():
            cursor.execute("INSERT INTO {} (name, phno) VALUES ('{}', '{}')".format(self.tname, k,v))
        print("Committed edits")
        conn.commit()
        conn.close()

# book1 = PhoneBook("inapp", "phonebook")
# book1.listContacts()
# book1.addContact("Kripa", "784512")
# book1.searchByName("Kripa")
# book1.searchByNumber("465132")
# book1.deleteContact("Karikku")
# book1.commit()
# del book1
# book1 = PhoneBook("inapp", "phonebook")
# book1.listContacts()


import torch
import numpy as np
import torch.nn as nn
import torch.optim as optim
import time
import os
import shutil
from datetime import datetime
from tqdm import tqdm
from load_data import load_data_imagefolder, load_hdf_data, load_split_data, load_fwa_data, load_split_data_random
from sklearn.metrics import confusion_matrix, roc_auc_score, classification_report
from transforms import (
    get_test_transform,
    train_albumentations,
    get_test_transform_albumentations,
)
import math
import matplotlib.pyplot as plt
from torch.utils.tensorboard import SummaryWriter
from hp import hp

# Disable if input sizes vary a lot
torch.backends.cudnn.benchmark = True

if not os.path.exists(hp.save_folder):
    os.mkdir(hp.save_folder)

now = datetime.now()
writer = None
if not (hp.quick_run or hp.pre_run):
    writer = SummaryWriter(os.path.join("runs", hp.model_name, now.strftime("%d%b%I:%M%p")))

device = hp.device

# sorted
classes = ["fake", "real"]

def load_multi_gpu(model):
    model.to(device)
    if False or torch.cuda.device_count() > 1:
        print("Using", torch.cuda.device_count(), "GPUs!")
        # torch.distributed.init_process_group(backend="nccl")
        # model = nn.parallel.DistributedDataParallel(model)
        model = nn.DataParallel(model, hp.device_ids)
    else:
        print(r"Multiple GPU's not available (╯°□°）╯︵ ┻━┻")
    return model

def train_model(
    model, datasets, dataloaders, criterion, optimizer, scheduler, num_epochs, save_loc
):
    since = time.time()

    best_acc = 0.0

    for epoch in range(num_epochs):
        if hp.use_cos_anneal_restart:
            optimizer = optim.SGD(model.parameters(), **hp.sgd_params)
            scheduler = optim.lr_scheduler.CosineAnnealingLR(
                optimizer, T_max=len(dataloaders["train"]), **hp.cos_anneal_sched_params
            )
        print(f"Epoch {epoch + 1}/{num_epochs}")
        print("-" * 10)
        print("LR", optimizer.param_groups[0]["lr"])

        probabilites = torch.zeros(0, dtype=torch.float32, device="cpu")
        predictions = torch.zeros(0, dtype=torch.long, device="cpu")
        true_val = torch.zeros(0, dtype=torch.long, device="cpu")

        # Each epoch has a training and validation phase
        for phase in ["train", "test"]:
            n_iter_this_phase = 0
            if phase == "train":
                model.train()  # Set model to training mode
            else:
                if hp.quick_run:
                    break
                model.eval()  # Set model to evaluate mode

            running_loss = 0.0
            running_corrects = 0

            for inputs, labels in tqdm(dataloaders[phase], ncols=0, mininterval=1):
                if hp.quick_run and n_iter_this_phase >= 150:
                    break
                n_iter_this_phase += 1

                non_blocking_transfer = (
                    True if phase == "train" and hp.use_pinned_memory_train else False
                )
                inputs = inputs.to(device, non_blocking=non_blocking_transfer)
                labels = labels.to(device, non_blocking=non_blocking_transfer)

                optimizer.zero_grad()

                with torch.set_grad_enabled(phase == "train"):
                    outputs = model(inputs)
                    _, preds = torch.max(outputs, 1)

                    loss = criterion(outputs, labels)

                    # backward + optimize only if in training phase
                    if phase == "train":
                        loss.backward()
                        optimizer.step()
                        if hp.use_one_cycle_lr or hp.use_cos_anneal_restart:
                            scheduler.step()
                    else:
                        # Softmax and calculate metrics
                        softmaxed = nn.functional.softmax(outputs, dim=1)
                        # Probability of class 1
                        probs = softmaxed[:, 1]
                        probabilites = torch.cat([probabilites, probs.view(-1).cpu()])
                        predictions = torch.cat([predictions, preds.view(-1).cpu()])
                        true_val = torch.cat([true_val, labels.view(-1).cpu()])

                # statistics
                running_loss += loss.item() * inputs.size(0)
                running_corrects += torch.sum(preds == labels.data)

            dataset_size = len(datasets[phase])
            epoch_loss = running_loss / dataset_size
            epoch_acc = float(running_corrects) / dataset_size
            if not hp.quick_run:
                writer.add_scalar(f"Loss/{phase}", epoch_loss, epoch)
                writer.add_scalar(f"Accuracy/{phase}", epoch_acc, epoch)

            print("{} Loss: {:.8f} Acc: {:.8f}".format(phase, epoch_loss, epoch_acc))

            if phase == "test":
                if hp.use_step_lr:
                    scheduler.step()
                elif hp.use_plateau_lr:
                    # Passing in test loss!
                    scheduler.step(epoch_loss)

            if phase == "test" and epoch_acc > best_acc:
                best_acc = epoch_acc
            if not hp.quick_run:
                file_name = f"{hp.model_name}_{epoch+1}.pt"
                torch.save(model.state_dict(), os.path.join(save_loc, file_name))
        if not hp.quick_run:
            true_val = true_val.numpy()
            predictions = predictions.numpy()
            probabilites = probabilites.numpy()
            print(classification_report(true_val, predictions, target_names=classes))
            writer.add_pr_curve("pr_curve", true_val, probabilites, epoch)
            print("Confusion Matrix")
            print(confusion_matrix(true_val, predictions))
            print()
            print("RoC AUC:")
            print(roc_auc_score(true_val, probabilites))
            print()
            probabilites = np.around(probabilites, decimals=1)
            hist, _ = np.histogram(probabilites, bins=3)
            print("# Predictions magnitude below 0.33", hist[0])
            print("# Predictions magnitude 0.33 to 0.66", hist[1])
            print("# Predictions magnitude above 0.66", hist[2])

    time_elapsed = time.time() - since
    print(
        "Training complete in {:.0f}m {:.0f}s".format(
            time_elapsed // 60, time_elapsed % 60
        )
    )
    print("Best val Acc: {:4f}".format(best_acc))

    # load best model weights
    return model

def load_data_for_model(model):
    image_size = model.get_image_size()
    mean, std = model.get_mean_std()
    train_data_transform = train_albumentations(image_size, mean, std)
    test_data_transform = get_test_transform_albumentations(image_size, mean, std)
    if hp.using_hdf:
        return load_hdf_data(hp.hdf_key)
    elif hp.using_split_random:
        return load_split_data_random(train_data_transform, test_data_transform)
    elif hp.using_split_cluster:
        return load_split_data(train_data_transform, test_data_transform)
    elif hp.using_fwa:
        return load_fwa_data(train_data_transform, test_data_transform)
    else:
        test_transform = get_test_transform(image_size, mean, std)
        data_transform = get_test_transform(image_size, mean, std)
        return load_data_imagefolder(
            train_data_transform=data_transform, test_data_transform=test_transform
        )

def pre_run():
    print("-------------------------------------")
    print(f"Now Pre-running model: {hp.model_name}")
    print("-------------------------------------")
    model = hp.model
    datasets, dataloaders = load_data_for_model(model)
    model = model.to(device)
    model = load_multi_gpu(model)
    weights = None
    if hp.use_class_weights:
        weights = torch.FloatTensor(hp.class_weights).to(device)
    criterion = nn.CrossEntropyLoss(weight=weights)

    # Find LR first
    logs, losses = find_lr(model, criterion, dataloaders["train"])
    torch.save(logs, "pre_run_logs.pt")
    torch.save(losses, "pre_run_losses.pt")
    # plt.plot(logs, losses)

def run():
    print("-------------------------------------")
    print(f"Now running model: {hp.model_name}")
    print(now.strftime("%A, %d %B at %I:%M %p"))
    print("-------------------------------------")

    save_loc = hp.model_name + now.strftime("%d%b%I:%M%p")
    save_loc = os.path.join(hp.save_folder, save_loc)
    if not hp.quick_run:
        if not os.path.isdir(save_loc):
            os.mkdir(save_loc)
        shutil.copy("hp.py", save_loc)
    model = hp.model
    datasets, dataloaders = load_data_for_model(model)

    print("Classes array (check order)")
    print(classes)

    model = model.to(device)
    model = load_multi_gpu(model)

    weights = None
    if hp.use_class_weights:
        weights = torch.FloatTensor(hp.class_weights).to(device)
    criterion = nn.CrossEntropyLoss(weight=weights)

    if hp.use_adamW:
        optimizer = optim.AdamW(model.parameters(), **hp.adamW_params)
    elif hp.use_sgd:
        optimizer = optim.SGD(model.parameters(), **hp.sgd_params)
    else:
        optimizer = None

    if hp.use_step_lr:
        scheduler = optim.lr_scheduler.StepLR(optimizer, **hp.step_sched_params)
    elif hp.use_plateau_lr:
        scheduler = optim.lr_scheduler.ReduceLROnPlateau(
            optimizer, **hp.plateau_lr_sched_params
        )
    elif hp.use_one_cycle_lr:
        scheduler = optim.lr_scheduler.OneCycleLR(
            optimizer, steps_per_epoch=len(dataloaders["train"]), **hp.oc_sched_params
        )
    else:
        scheduler = None

    model = train_model(
        model,
        datasets,
        dataloaders,
        criterion,
        optimizer,
        scheduler,
        hp.num_epochs,
        save_loc,
    )

def find_lr(net, criterion, trn_loader, init_value=1e-8, final_value=10.0, beta=0.98):
    """
    https://sgugger.github.io/how-do-you-find-a-good-learning-rate.html
    https://sgugger.github.io/the-1cycle-policy.html
    logs,losses = find_lr()
    plt.plot(logs,losses)
    For a OneCycleLR, The maximum should be the value picked with the Learning Rate Finder, and the lower one can be ten times lower.
    """
    if hp.use_adamW:
        optimizer = optim.AdamW(net.parameters(), **hp.adamW_params)
    elif hp.use_sgd:
        optimizer = optim.SGD(net.parameters(), **hp.sgd_params)
    num = len(trn_loader) - 1
    mult = (final_value / init_value) ** (1 / num)
    lr = init_value
    optimizer.param_groups[0]["lr"] = lr
    avg_loss = 0.0
    best_loss = 0.0
    batch_num = 0
    losses = []
    log_lrs = []
    for inputs, labels in tqdm(trn_loader, ncols=0, mininterval=1):
        batch_num += 1
        inputs = inputs.to(device, non_blocking=hp.use_pinned_memory_train)
        labels = labels.to(device, non_blocking=hp.use_pinned_memory_train)

        optimizer.zero_grad()
        outputs = net(inputs)
        loss = criterion(outputs, labels)

        # Compute the smoothed loss
        avg_loss = beta * avg_loss + (1 - beta) * loss.item()
        smoothed_loss = avg_loss / (1 - beta ** batch_num)

        # Stop if the loss is exploding
        if batch_num > 1 and smoothed_loss > 4 * best_loss:
            return log_lrs, losses

        # Record the best loss
        if smoothed_loss < best_loss or batch_num == 1:
            best_loss = smoothed_loss

        # Store the values
        losses.append(smoothed_loss)
        log_lrs.append(math.log10(lr))

        # Do the SGD step
        loss.backward()
        optimizer.step()

        # Update the lr for the next step
        lr *= mult
        optimizer.param_groups[0]["lr"] = lr
    return log_lrs[10:-5], losses[10:-5]

if __name__ == "__main__":
    if hp.pre_run:
        pre_run()
    else:
        run()
    writer.close()


from twisted.python.runtime import platform
from twisted.internet.test.reactormixins import ReactorBuilder
from twisted.internet.protocol import Protocol
if not platform.isWindows():
    from twisted.internet._posixstdio import StandardIO

class StdioFilesTests(ReactorBuilder):
    """
    L{StandardIO} supports reading and writing to filesystem files.
    """

    def setUp(self):
        path = self.mktemp()
        file(path, "w").close()
        self.extraFile = file(path, "r+")

    def test_addReader(self):
        """
        Adding a filesystem file reader to a reactor will make sure it is
        polled.
        """
        reactor = self.buildReactor()

        class DataProtocol(Protocol):
            data = ""
            def dataReceived(self, data):
                self.data += data
                # It'd be better to stop reactor on connectionLost, but that
                # fails on FreeBSD, probably due to
                # http://bugs.python.org/issue9591:
                if self.data == "hello!":
                    reactor.stop()

        path = self.mktemp()
        f = file(path, "w")
        f.write("hello!")
        f.close()
        f = file(path, "r")

        # Read bytes from a file, deliver them to a protocol instance:
        protocol = DataProtocol()
        StandardIO(protocol, stdin=f.fileno(),
                   stdout=self.extraFile.fileno(),
                   reactor=reactor)

        self.runReactor(reactor)
        self.assertEqual(protocol.data, "hello!")

    def test_addWriter(self):
        """
        Adding a filesystem file writer to a reactor will make sure it is
        polled.
        """
        reactor = self.buildReactor()

        class DisconnectProtocol(Protocol):
            def connectionLost(self, reason):
                reactor.stop()

        path = self.mktemp()
        f = file(path, "w")

        # Write bytes to a transport, hopefully have them written to a file:
        protocol = DisconnectProtocol()
        StandardIO(protocol, stdout=f.fileno(),
                   stdin=self.extraFile.fileno(), reactor=reactor)
        protocol.transport.write("hello")
        protocol.transport.write(", world")
        protocol.transport.loseConnection()

        self.runReactor(reactor)
        f.close()
        f = file(path, "r")
        self.assertEqual(f.read(), "hello, world")
        f.close()

    def test_removeReader(self):
        """
        Removing a filesystem file reader from a reactor will make sure it is
        no longer polled.
        """
        reactor = self.buildReactor()
        self.addCleanup(self.unbuildReactor, reactor)

        path = self.mktemp()
        file(path, "w").close()
        # Cleanup might fail if file is GCed too soon:
        self.f = f = file(path, "r")

        # Have the reader added:
        stdio = StandardIO(Protocol(), stdin=f.fileno(),
                           stdout=self.extraFile.fileno(),
                           reactor=reactor)
        self.assertIn(stdio._reader, reactor.getReaders())
        stdio._reader.stopReading()
        self.assertNotIn(stdio._reader, reactor.getReaders())

    def test_removeWriter(self):
        """
        Removing a filesystem file writer from a reactor will make sure it is
        no longer polled.
        """
        reactor = self.buildReactor()
        self.addCleanup(self.unbuildReactor, reactor)

        # Cleanup might fail if file is GCed too soon:
        self.f = f = file(self.mktemp(), "w")

        # Have the reader added:
        protocol = Protocol()
        stdio = StandardIO(protocol, stdout=f.fileno(),
                           stdin=self.extraFile.fileno(),
                           reactor=reactor)
        protocol.transport.write("hello")
        self.assertIn(stdio._writer, reactor.getWriters())
        stdio._writer.stopWriting()
        self.assertNotIn(stdio._writer, reactor.getWriters())

    def test_removeAll(self):
        """
        Calling C{removeAll} on a reactor includes descriptors that are
        filesystem files.
        """
        reactor = self.buildReactor()
        self.addCleanup(self.unbuildReactor, reactor)

        path = self.mktemp()
        file(path, "w").close()
        # Cleanup might fail if file is GCed too soon:
        self.f = f = file(path, "r")

        # Have the reader added:
        stdio = StandardIO(Protocol(), stdin=f.fileno(),
                           stdout=self.extraFile.fileno(), reactor=reactor)
        # And then removed:
        removed = reactor.removeAll()
        self.assertIn(stdio._reader, removed)
        self.assertNotIn(stdio._reader, reactor.getReaders())

    def test_getReaders(self):
        """
        C{reactor.getReaders} includes descriptors that are filesystem files.
        """
        reactor = self.buildReactor()
        self.addCleanup(self.unbuildReactor, reactor)

        path = self.mktemp()
        file(path, "w").close()
        # Cleanup might fail if file is GCed too soon:
        self.f = f = file(path, "r")

        # Have the reader added:
        stdio = StandardIO(Protocol(), stdin=f.fileno(),
                           stdout=self.extraFile.fileno(), reactor=reactor)
        self.assertIn(stdio._reader, reactor.getReaders())

    def test_getWriters(self):
        """
        C{reactor.getWriters} includes descriptors that are filesystem files.
        """
        reactor = self.buildReactor()
        self.addCleanup(self.unbuildReactor, reactor)

        # Cleanup might fail if file is GCed too soon:
        self.f = f = file(self.mktemp(), "w")

        # Have the reader added:
        stdio = StandardIO(Protocol(), stdout=f.fileno(),
                           stdin=self.extraFile.fileno(), reactor=reactor)
        self.assertNotIn(stdio._writer, reactor.getWriters())
        stdio._writer.startWriting()
        self.assertIn(stdio._writer, reactor.getWriters())

    if platform.isWindows():
        skip = ("StandardIO does not accept stdout as an argument to Windows.  "
                "Testing redirection to a file is therefore harder.")

globals().update(StdioFilesTests.makeTestCaseClasses())


"""Run the code with self-healing."""

import subprocess
import time
from pathlib import Path

import fire

from prompt_templates.information_retrieval import find_file_with_error_template
from prompt_templates.code_generation import fix_code_template
from utils.llm import send_templated_message_to_llm, query_llm
from utils.file_io import read_file, write_file, compute_diffs, get_main_script_name
from utils.log import log
from config import project_configs

def run_script_inside_subprocess_with_timeout(
    root_folder_name: str,
    bash_script: str,
    timeout: str | None = None
) -> tuple[str | None, str | None]:
    """Run the script inside a subprocess with a timeout."""
    start = time.time()
    process = subprocess.Popen([f"./{bash_script}"],
                               cwd=root_folder_name,
                               stdout=subprocess.PIPE,
                               stderr=subprocess.PIPE)
    while True:
        if process.poll() is not None:  # If the process has finished
            break
        if timeout is not None:
            if time.time() - start > timeout:  # If timeout has passed
                process.terminate()
                log.debug(f"Script was terminated after {timeout} seconds.")
                return None, None
        time.sleep(0.1)  # Sleep for a short time to prevent busy waiting

    stdout, stderr = process.communicate()

    return stdout, stderr

def run_with_self_healing_code(
    project_name: str,
    max_heal_attempts: int = 25,
    max_improvement_iterations_per_llm_query: int = 3
) -> None:
    """Run the code with self-healing.

    Args:
        project_path: The path to the project.
        max_fix_attempts: The maximum number of attempts to fix the code.
        max_improvement_iterations_per_llm_query:
            The maximum number of improvement iterations per LLM query.
    """
    project_requirements = read_file(Path(project_configs[project_name]["requirements_document"]))
    tech_spec = read_file(
        Path(project_configs[project_name]["project_path"]) / Path("technical_specification.md"))
    root_folder_name = project_configs[project_name]["project_path"]
    bash_script = get_main_script_name(root_folder_name)
    subprocess.run(["chmod", "+x", str(Path(root_folder_name) / Path(bash_script))])
    code_errors_out = True
    attempts = 1
    while code_errors_out and attempts <= max_heal_attempts:
        stdout, stderr = run_script_inside_subprocess_with_timeout(
            root_folder_name, bash_script)

        if stderr is None:
            code_errors_out = False
            log.info("Script ran successfully.")
        else:
            log.debug(f"Output from stdout: {stdout}")
            log.debug(f"Output from stderr: {stderr}")
            message_to_send = find_file_with_error_template(tech_spec, stdout, stderr)
            reply = query_llm(message_to_send)
            if reply == "There is no error.":
                code_errors_out = False
                log.info("Script ran successfully.")
            else:
                path_and_file_name = reply
                code = read_file(path_and_file_name)
                message_to_send = fix_code_template(
                    project_requirements, tech_spec, path_and_file_name, code, stderr)
                content = send_templated_message_to_llm(
                    message_to_send, max_improvement_iterations_per_llm_query)
                log.info(compute_diffs(code, content))
                write_file(path_and_file_name, content)
                attempts += 1

if __name__ == "__main__":
    fire.Fire(run_with_self_healing_code)


from bs4 import BeautifulSoup
import fileinput
import html2text
import json
import logging
import markdown
import random
import requests
import shutil
import sys

# Load json in memory
def get_challenge_updates(file_path: str, update_field: str) -> list:
    # open with utf 8 encoding

    with open(file_path, 'r') as file:
        data = json.load(file)

    temp = list()
    for i in data[update_field]:
        temp.append(i)

    file.close()
    print("⚙Updating challenges...")
    return temp

# Choose a random challenge
def select_daily_challenge(file: str, upcoming_list: list, old_list: list) -> str:
    daily_challenge = random.choice(upcoming_list)
    if len(old_list) >= len(upcoming_list):
        old_list = []
        update_challenges_list(file, old_list)
        daily_challenge = random.choice(upcoming_list)
        print("🌟 " + daily_challenge)
        return daily_challenge

    while is_in_old(daily_challenge, old_list):
        print("Oops! Already taken 🎯")
        daily_challenge = random.choice(upcoming_list)

    old_list.append(daily_challenge)
    update_challenges_list(file, old_list)
    print("🌟 " + daily_challenge)
    return daily_challenge

def is_in_old(i: str, old_list: list) -> bool:
    return i in old_list

# === Complementary method
def update_challenges_list(file, new_old: list):
    with open(file, "r+", encoding="utf-8") as json_file:
        data = json.load(json_file)
        data['old'] = new_old

        json_file.seek(0)  # rewind
        json.dump(data, json_file, indent=2)
        json_file.truncate()
        print("Old list update: ⏬")
        print(new_old)

# Get today's challenge title and content
def get_challenge_content(file: str, challenge: str) -> str:
    with open(file, "r+") as json_file:
        data = json.load(json_file)

        for i in data['challenges']:
            name = i['title']
            if name == challenge:
                content = i['content']
                print("👀 " + content)
                return content

# Get last challenge
def get_last_challenge(file: str) -> str:
    file = open(file, "rb")
    data = json.load(file)
    last = data['last_problem_title']
    print("👴 Latest challenge: " + last)
    file.close()
    return last

def set_last_challenge(file, challenge):
    with open(file, "r+") as json_file:
        data = json.load(json_file)
        data['last_problem_title'] = challenge
        json_file.seek(0)
        json.dump(data, json_file)
        json_file.truncate()
        print("✅ Last challenge updated: " + challenge)
    json_file.close()

def set_last_challenge_content(file: str, today_challenge_content: str):
    with open(file, "r+", encoding="utf-8") as json_file:
        data = json.load(json_file)
        data['last_problem_content'] = today_challenge_content
        json_file.seek(0)
        json.dump(data, json_file)
        json_file.truncate()
        print("✅ Last challenge content updated: " + str(today_challenge_content))
    json_file.close()

# Write today's challenge and solution
def write_challenge_of_the_day(readme_file: str, new_text: str, placeholder):
    # write to markdown
    for line in fileinput.input(readme_file, inplace=True, encoding="utf-8"):
        if placeholder in line:
            line = line.replace(placeholder, new_text)
        sys.stdout.write(line)

def write_solution(solution, lang):
    pass

def main():
    # Make a GET request to the problem page
    url: str = "https://leetcode.com/problems/all/"
    page = requests.get(url)

    # Parse the HTML of the page
    soup = BeautifulSoup(page.content, "html.parser")

    # Find the title and description elements
    title_element = soup.find("h4", class_="title")
    description_element = soup.find("div", class_="question-content")

    # Extract the text from the elements
    title = title_element
    description = description_element

    # Save the information to a JSON file
    data = {
        "title": title,
        "description": description
    }
    with open("problem.json", "w") as f:
        json.dump(data, f)

# -------------
def read_file(file_name: str):
    logging.info("📰Reading file...")
    with open(file_name, "r+", encoding="utf-8") as file:
        text = file.read()
        return str(text)

def save_text_to_html(text):
    html = markdown.markdown(text)
    with open("scripts/readme.html", "w", encoding="utf-8") as file:
        file.write(html)

def replace_text_in_html(class_name: str, tag: str, new_text: str):
    with open("scripts/readme.html", "r+", encoding="utf-8") as file:
        html_text = file.read()
        soup = BeautifulSoup(html_text, features="html.parser")
        element = soup.find(tag, {"class": class_name})
        print("🔎 Tag found")
        print(element.string)
        element.string = new_text

        file.seek(0)
        file.write(soup.prettify())
        file.truncate()
        print("✅ Text replaced in html")

def convert_html_to_markdown(html):
    markdown_text = html2text.html2text(html)
    with open("README.md", "w+", encoding="utf-8") as file:
        file.write(markdown_text)

def copyfile(file_to_copy, new_path):
    shutil.copyfile(file_to_copy, new_path)

if __name__ == '__main__':
    # Variables
    challenges_json = "scripts/challenges.json"

    # TODO: Convert the current README into HTML daily

    # Load lists in memory
    upcoming = get_challenge_updates(challenges_json, "upcoming")
    old = get_challenge_updates(challenges_json, "old")

    # Choose a random challenge ⬇
    # Challenge Name
    daily = select_daily_challenge(challenges_json, upcoming, old)

    # Challenge Content
    daily_content = get_challenge_content(challenges_json, daily)

    # Pull the HTML temp file and replace the text
    replace_text_in_html("problem-title", "h3", daily)
    replace_text_in_html("problem-content", "p", daily_content)
    replace_text_in_html("problem-solution", "pre", "🤓")

    # Convert it back to markdown
    convert_html_to_markdown(read_file("scripts/readme.html"))

    # Copy the README to the root folder to replace it
    copyfile("scripts/readme.md", "../README.md")

    logging.info('Done!')