def start():
    for y in range(N):
        for x in range(N):
            if maze[y][x] == 2:
                return [y, x]

def maze_run():
    while queue:
        now = queue.pop(0)
        y = now[0]
        x = now[1]
        for i in range(4):
            ny = y + dy[i]
            nx = x + dx[i]
            if 0 <= ny < N and 0 <= nx < N:
                if maze[ny][nx] != 1 and visited[ny][nx] == 0:
                    visited[ny][nx] = visited[y][x] + 1
                    queue.append((ny, nx))
                    if maze[ny][nx] == 3:
                        return visited[ny][nx] - 2
    return 0

dy = [-1, 0, 1, 0]
dx = [0, 1, 0, -1]

T = int(input())
for tc in range(T):
    N = int(input())
    maze = [list(map(int, list(input()))) for _ in range(N)]
    visited = [[0] * N for _ in range(N)]
    queue = []
    st = start()
    queue.append(st)
    visited[st[0]][st[1]] = 1
    ans = maze_run()

    print('#{} {}'.format(tc+1, ans))


from fastapi import APIRouter, Depends, Request, status, HTTPException
from fastapi.responses import JSONResponse
from fastapi.encoders import jsonable_encoder
from sqlalchemy.orm import Session
from fastapi.security import HTTPBearer
from db import get_db
import crud
import models as mod

addition_router = APIRouter(tags=['Addition'], dependencies=[Depends(HTTPBearer())])

@addition_router.post('/api/create-addition')
async def create_addition(header_param: Request, req: mod.AdditionSchema, db: Session = Depends(get_db)):
    result = await crud.create_addition(header_param, req, db)
    if result == -1:
        return HTTPException(status_code=status.HTTP_401_UNAUTHORIZED)
    if result:
        result = jsonable_encoder(result)
        result['msg'] = 'Создано!'
        return JSONResponse(content=result, status_code=status.HTTP_201_CREATED)
    else:
        return HTTPException(status_code=status.HTTP_204_NO_CONTENT)

@addition_router.put('/api/update-addition/{id}')
async def update_addition(id: int, header_param: Request, req: mod.AdditionSchema, db: Session = Depends(get_db)):
    result = await crud.update_addition(id, header_param, req, db)
    if result == -1:
        return HTTPException(status_code=status.HTTP_401_UNAUTHORIZED)
    if result:
        result = jsonable_encoder(result)
        return JSONResponse(content=result, status_code=status.HTTP_200_OK)
    else:
        return HTTPException(status_code=status.HTTP_204_NO_CONTENT)


""" Module containing pipelines (reoccuring function calls) for use with apscheduler. """

from datetime import datetime
import logging
from requests import Session
import json
from apscheduler.schedulers.background import BlockingScheduler

from driver.driver_config_builder import DriverConfig
from driver.compute_server_client import ComputeServerClient
from driver.database import (
    collect_db_level_observation_for_on_prem,
    collect_table_level_observation_for_on_prem,
)
from datetime import datetime

TUNE_JOB_ID = "tune_job"
DB_LEVEL_MONITOR_JOB_ID = "db_level_monitor_job"
APPLY_EVENT_JOB_ID = "apply_event_job"
TABLE_LEVEL_MONITOR_JOB_ID = "table_level_monitor_job"

def driver_pipeline(
    config: DriverConfig,
    job_id: str,  # pylint: disable=unused-argument
) -> None:
    """
    Run the core pipeline for the driver deployment
    """
    logging.info("Running driver pipeline deployment!")

    #compute_server_client = ComputeServerClient(
    #    config.server_url, Session(), config.api_key
    #)

    if job_id == DB_LEVEL_MONITOR_JOB_ID:
        _db_level_monitor_driver_pipeline_for_on_prem(config)
    elif job_id == TABLE_LEVEL_MONITOR_JOB_ID:
        _table_level_monitor_driver_pipeline_for_on_prem(config)
    # elif job_id == LINUX

def _db_level_monitor_driver_pipeline_for_on_prem(
    config: DriverConfig,
    #compute_server_client: ComputeServerClient,
) -> None:
    """
    Regular monitoring pipeline that collects database level metrics and configs every minute

    Args:
        config: Driver configuration.
        compute_server_client: Client interacting with server in Ottertune.
    Raises:
        DriverException: Driver error.
        Exception: Other unknown exceptions that are not caught as DriverException.
    """
    logging.debug("Collecting db level observation data.")
    db_level_observation = collect_db_level_observation_for_on_prem(config)
    now = datetime.now()
    file_name = now.strftime('%Y%m%d_%H%M%S')
    f_path = open('path.txt' , 'r' )
    path = f_path.readline()
    path.rstrip('\n')
    with open(path+'/'+file_name, 'w') as outfile:
        json.dump(db_level_observation, outfile)
    logging.debug("Saving db level observation data to the server.")

    #compute_server_client.post_db_level_observation(db_level_observation)

def _table_level_monitor_driver_pipeline_for_on_prem(
    config: DriverConfig,
    #compute_server_client: ComputeServerClient,
) -> None:
    """
    Regular monitoring pipeline that collects table level metrics every hour

    Args:
        config: Driver configuration.
        comppute_server_client: Client interacting with server in Ottertune.
    Raises:
        DriverException: Driver error.
        Exception: Other unknown exceptions that are not caught as DriverException.
    """
    logging.debug("Collecting table level observation data")
    table_level_observation = collect_table_level_observation_for_on_prem(config)

    logging.debug("Posting table level observation data to the server.")
    #compute_server_client.post_table_level_observation(table_level_observation)

def _get_interval(config: DriverConfig, job_id: str) -> int:
    """Get the scheduled time interval (sec) based on job id."""

    if job_id == DB_LEVEL_MONITOR_JOB_ID:
        interval_s = int(config['monitor_interval'])
    elif job_id == TABLE_LEVEL_MONITOR_JOB_ID:
        interval_s = int(config['table_level_monitor_interval'])
    else:
        raise ValueError(f"Job {job_id} is not supported")
    return interval_s

def _start_job(
    scheduler: BlockingScheduler, config: DriverConfig, job_id: str, interval: int
) -> None:
    "Helper to start new job"
    logging.info("Initializing driver pipeline (job %s)...", job_id)

    kwargs = {}
    if job_id in (DB_LEVEL_MONITOR_JOB_ID, TABLE_LEVEL_MONITOR_JOB_ID):
        kwargs["next_run_time"] = datetime.now()

    scheduler.add_job(
        driver_pipeline,
        "interval",
        seconds=interval,
        args=[config, job_id],
        id=job_id,
        **kwargs,
    )
    logging.info("Running driver pipeline every %d seconds (job %s).", interval, job_id)

def _update_job(
    scheduler: BlockingScheduler,
    old_config: DriverConfig,
    new_config: DriverConfig,
    job_id: str,
    interval: int,
) -> None:
    "Helper to update pre-existing job"
    logging.info("Found new config (job %s)...", job_id)
    # grab old interval before modification
    old_interval = _get_interval(old_config, job_id)
    scheduler.modify_job(job_id, args=[new_config, job_id])
    if old_interval != interval:
        scheduler.reschedule_job(job_id, trigger="interval", seconds=interval)
        logging.info(
            "Running driver pipeline every %d seconds (job %s).", interval, job_id
        )

def schedule_or_update_job(
    scheduler: BlockingScheduler, config: DriverConfig, job_id: str
) -> None:
    """
    Apply configuration change to the job. If the configuration does not change, it will do nothing.
    If the job is not scheduled, it will start a job.

    Args:
        config: Driver configuration.
        job_id: Job ID.
    Raises:
        DriverException: Driver error.
        Exception: Other unknown exceptions that are not caught as DriverException.
    """
    interval = _get_interval(config, job_id)
    job = scheduler.get_job(job_id)

    if not job:
        # NB: first invocation is at current_time + interval
        _start_job(scheduler=scheduler, config=config, job_id=job_id, interval=interval)
    else:
        old_config = job.args[0]
        if old_config != config:
            _update_job(
                scheduler=scheduler,
                old_config=old_config,
                new_config=config,
                job_id=job_id,
                interval=interval,
            )


#!/usr/bin/env python3
# coding: utf-8
"""
@author: Ping Qiu  qiuping1@genomics.cn
@last modified by: Ping Qiu
@file: test_data_helper.py
@time: 2021/03/16
"""

import sys

sys.path.append('/data/workspace/st/stereopy-release')

from stereo.utils.data_helper import select_group
import pandas as pd
from anndata import AnnData
import numpy as np

np.random.seed(9)

def init_andata(genes=50, cells=200):
    gname = [f'g{i}' for i in range(genes)]
    cname = [f'c{i}' for i in range(cells)]
    groups = np.random.choice(['1', '2', '3'], cells)
    x = np.random.randint(0, 100, (cells, genes))
    var = pd.DataFrame(index=gname)
    obs = pd.DataFrame(index=cname)
    obs['marker_genes'] = groups
    andata = AnnData(x, obs=obs, var=var)
    return andata

andata = init_andata()
print(andata.obs[andata.obs['marker_genes'] == '1'])
data = select_group(andata, groups='1', clust_key='marker_genes')
print(data)


from whisper_jax import FlaxWhisperPipline
import jax.numpy as jnp
import time
from jax.experimental.compilation_cache import compilation_cache as cc
import os
import jax

start_time = time.time()

# environment variable
os.environ["XLA_FLAGS"] = "--xla_gpu_strict_conv_algorithm_picker=false"

# change from GPU to CPU
jax.config.update('jax_platform_name', 'cpu')
print(f"Device: {jax.numpy.ones(3).device()}") # TFRT_CPU_0

# speed-up the compilation time if we close our kernel and want to compile the model again
cc.initialize_cache("./jax_cache")

# instantiate pipeline with float16 and enable batching
pipeline = FlaxWhisperPipline("openai/whisper-large-v2",
                            #   dtype=jnp.float16, # For most GPUs, the dtype should be set to jnp.float16. For A100 GPUs or TPUs, the dtype should be set to jnp.bfloat16
                            #   batch_size=1 # see https://github.com/sanchit-gandhi/whisper-jax#batching
                              )

# transcribe and return timestamps
outputs = pipeline("Animal Communication Ezekiel.m4a",  task="transcribe", return_timestamps=True)
text = outputs["text"]
chunks = outputs["chunks"]

# Time calculation
time_taken_seconds = time.time() - start_time # Calculate the time taken in seconds
time_taken_formatted = "{:.3f}".format(time_taken_seconds) # Format the time taken to three decimal places

# Calculate hours, minutes, and seconds
hours = int(time_taken_seconds // 3600)
minutes = int((time_taken_seconds % 3600) // 60)
seconds = int(time_taken_seconds % 60)

# Print the time taken in hours, minutes, and seconds format with three decimal places
print(
    "Time taken: {} hours, {} minutes, {} seconds".format(
        hours, minutes, time_taken_formatted
    )
)


from .base import LLMRequestProcessor
from .helpers import map_substring_indices, find_sentences

class EncodeRequestProcessor(LLMRequestProcessor):
    def __init__(self, config, tokenizer):
        super().__init__(config)
        self.tokenizer = tokenizer

    def process(self, body):
        """
        Input:
            {
                "text": "This is a sentence"
            }

        Output:
            {
                "token_ids": [1, 54, 23, ..., 2],
                "token_mapping": [[0, 4], [5, 7], [8, 9], [10, 18]],
                "sentence_mapping": [[0, 19], [20, 39]]
            }
        """

        text = body["text"]

        # encode text into a list of token id's
        token_ids = self.tokenizer.encode(text, return_tensors="pt").squeeze(0).tolist()

        # split text into tokens in text form
        token_text_list = [self.tokenizer.decode(token_id) for token_id in token_ids]

        # map token id's to their corresponding indices in the original text
        token_mapping = map_substring_indices(text, token_text_list)
        sentence_mapping = map_substring_indices(text, find_sentences(text))

        return {
            "token_ids": token_ids,
            "token_mapping": token_mapping,
            "sentence_mapping": sentence_mapping
        }

class DecodeRequestProcessor(LLMRequestProcessor):
    def __init__(self, config, tokenizer):
        super().__init__(config)
        self.tokenizer = tokenizer
        self.space_char = self.config["SPC"]

    def process(self, body):
        """
        Input:
            {
                "token_ids": [1, 54, 23, ..., 2]
            }

        Output:
            {
                "text": "This is a sentence",
                "token_mapping": [[0, 4], [5, 7], [8, 9], [10, 18]],
                "sentence_mapping": [[0, 19], [20, 39]]
            }
        """

        token_ids = body["token_ids"]

        # split text into tokens in text form
        token_text_list = self.tokenizer.convert_ids_to_tokens(token_ids)

        # join tokens into a single string
        token_text = "".join(token_text_list).replace(self.space_char, " ").strip()

        # remove special tokens
        token_text_list = [s.replace(self.space_char, "") for s in token_text_list]

        # map token id's to their corresponding indices in the final text
        token_mapping = map_substring_indices(token_text, token_text_list)
        sentence_mapping = map_substring_indices(token_text, find_sentences(token_text))

        return {
            "text": token_text,
            "token_mapping": token_mapping,
            "sentence_mapping": sentence_mapping
        }


import keras
import numpy as np
import math
from keras.layers import Dense
from keras.layers import Input
from keras.layers import Conv2D
from keras.layers import add
from keras.layers import Activation
from keras.layers import GlobalAveragePooling2D
from keras.models import Model
from keras.datasets import cifar10
from keras.callbacks import Callback
from keras.callbacks import LearningRateScheduler, ModelCheckpoint
from keras import optimizers
from keras.preprocessing.image import ImageDataGenerator
from keras import regularizers
from keras.layers import BatchNormalization
import tensorflow as tf
from keras import backend as K
from keras.backend.tensorflow_backend import set_session

'''
Tensorflow backend default
'''

config = tf.ConfigProto()         #using GPU
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)

STACK_NUM = 5
NUM_CLASSES = 10
WIDTH = 32
HEIGHT = 32
BATCH_SIZE = 256       #decrease the number if you run out your GPU memory
EPOCHES = 500
ITERATIONS = 50000//BATCH_SIZE +1
WEIGHT_DECAY = 1e-4           #according to the paper
FILE_PATH = '/cluster/home/it_stu25/dllab/model/best_model.h5'
BASE_LR = 0.1

class cosine(Callback):
    def __init__(self,
        base_lr,
        steps_per_epoch,
        warm_up_epoches
        ):
        super(cosine, self).__init__()
        self.base_lr = base_lr
        self.steps_per_epoch = steps_per_epoch
        self.warm_up_epoches = warm_up_epoches
        self.WarmUp = 0
        self.step_n = 0

    def on_epoch_begin(self, epoch, logs=None):
        if(epoch<self.warm_up_epoches):
            self.WarmUp=1
            self.step_n = epoch*(self.steps_per_epoch)
        else:
            self.WarmUp = 0
            self.step_n = (epoch-self.warm_up_epoches)*(self.steps_per_epoch)

    def on_batch_begin(self, batch, logs=None):
        if (self.WarmUp):
            lr = (batch+self.step_n)*(self.base_lr)/(self.steps_per_epoch*self.warm_up_epoches)
            K.set_value(self.model.optimizer.lr, lr)
        else:
            lr = 0.5 * self.base_lr * ( 1 + math.cos( math.pi * (batch + self.step_n)/(self.steps_per_epoch*(EPOCHES - self.warm_up_epoches))))
            K.set_value(self.model.optimizer.lr, lr)

def residual_block(x, filters, increase=False):
    stride = (1, 1)
    if increase:
        stride = (2, 2)

    o1 = Activation('relu')(BatchNormalization(momentum=0.9, epsilon=1e-5)(x))
    conv_1 = Conv2D(filters, kernel_size=(3, 3), strides=stride, padding='same',
                    kernel_initializer="he_normal",
                    kernel_regularizer=regularizers.l2(WEIGHT_DECAY))(o1)
    o2 = Activation('relu')(BatchNormalization(momentum=0.9, epsilon=1e-5)(conv_1))
    conv_2 = Conv2D(filters, kernel_size=(3, 3), strides=(1, 1), padding='same',
                    kernel_initializer="he_normal",
                    kernel_regularizer=regularizers.l2(WEIGHT_DECAY))(o2)
    if increase:
        projection = Conv2D(filters, kernel_size=(1, 1), strides=(2, 2), padding='same',
                            kernel_initializer="he_normal",
                            kernel_regularizer=regularizers.l2(WEIGHT_DECAY))(o1)
        block = add([conv_2, projection])
    else:
        block = add([conv_2, x])
    return block

def residual_network(input_tensor, stack_num = 5):
    x = Conv2D(filters=16, kernel_size=(3, 3), strides=(1, 1), padding='same',
               kernel_initializer="he_normal",
               kernel_regularizer=regularizers.l2(WEIGHT_DECAY))(input_tensor)

    for _ in range(stack_num):
        x = residual_block(x, 16, False)

    # input: 32x32x16 output: 16x16x32
    x = residual_block(x, 32, True)
    for _ in range(1, stack_num):
        x = residual_block(x, 32, False)

    # input: 16x16x32 output: 8x8x64
    x = residual_block(x, 64, True)
    for _ in range(1, stack_num):
        x = residual_block(x, 64, False)

    x = BatchNormalization(momentum=0.9, epsilon=1e-5)(x)
    x = Activation('relu')(x)
    x = GlobalAveragePooling2D()(x)

    # input: 64 output: 10
    x = Dense(10, activation='softmax', kernel_initializer="he_normal",
              kernel_regularizer=regularizers.l2(WEIGHT_DECAY))(x)
    return x

def color_preprocessing(x_train, x_test):
    x_train = x_train.astype('float32')
    x_test = x_test.astype('float32')
    mean = [125.307, 122.95, 113.865]
    std = [62.9932, 62.0887, 66.7048]
    for i in range(3):
        x_train[:, :, :, i] = (x_train[:, :, :, i] - mean[i]) / std[i]
        x_test[:, :, :, i] = (x_test[:, :, :, i] - mean[i]) / std[i]
    return x_train, x_test

if __name__ == '__main__':

    (x_train, y_train),(x_test,y_test) = cifar10.load_data()
    y_train = keras.utils.to_categorical(y_train, 10)
    y_test = keras.utils.to_categorical(y_test, 10)

    x_train, x_test = color_preprocessing(x_train, x_test)
    img_input = Input(shape=(HEIGHT,WIDTH,3))     #channel_last
    output = residual_network(img_input, STACK_NUM)
    resnet = Model(img_input, output)
    #other optimizers may achieve better performance
    sgd = optimizers.SGD(lr=0.1, momentum=0.9, nesterov=True)     #according the paper
    resnet.compile(loss='categorical_crossentropy',optimizer=sgd, metrics=['accuracy'])
    checkpoint = ModelCheckpoint(FILE_PATH, monitor='val_acc', verbose=1, save_best_only=True,mode='max')
    cbks = [cosine(base_lr = BASE_LR, steps_per_epoch = ITERATIONS,warm_up_epoches = 5),checkpoint]
    datagen = ImageDataGenerator(horizontal_flip=True,
                                 width_shift_range=0.125,
                                 height_shift_range=0.125,
                                 fill_mode='constant', cval=0.)
    datagen.fit(x_train)

    resnet.fit_generator(datagen.flow(x_train,y_train,batch_size=BATCH_SIZE),
                         steps_per_epoch=ITERATIONS,
                         epochs=EPOCHES,
                         callbacks=cbks,
                         validation_data=(x_test,y_test))

    resnet.save("resnet_baseline.h5")


# https://onlinejudge.u-aizu.ac.jp/courses/library/4/CGL/all/CGL_2_A
# 内積で判別すれば良い
# 丸め込み誤差の判別が糞すぎてもうやりたくない

from math import sqrt, isclose

def euclid_norm(x, y):
    return sqrt(pow(x, 2) + pow(y, 2))

def is_parallel(x0, y0, x1, y1, x2, y2, x3, y3):
    x_v1 = x1 - x0
    y_v1 = y1 - y0
    x_v2 = x3 - x2
    y_v2 = y3 - y2
    dot = (x_v1 * x_v2 + y_v1 * y_v2) / \
        (euclid_norm(x_v1, y_v1) * euclid_norm(x_v2, y_v2))
    # print(dot)
    if isclose(dot, 1, rel_tol=1e-10) or isclose(dot, -1, rel_tol=1e-10):
        return True
    return False

def is_orhogonal(x0, y0, x1, y1, x2, y2, x3, y3):
    x_v1 = x1 - x0
    y_v1 = y1 - y0
    x_v2 = x3 - x2
    y_v2 = y3 - y2
    dot = (x_v1 * x_v2 + y_v1 * y_v2) / \
        (euclid_norm(x_v1, y_v1) * euclid_norm(x_v2, y_v2))
    # print(dot)
    if isclose(dot, 0, rel_tol=1e-10):
        return True
    return False

# load data
N = int(input())
for i in range(N):
    tmp = list(map(int, input().split()))
    if is_parallel(*tmp):
        print(2)
    elif is_orhogonal(*tmp):
        print(1)
    else:
        print(0)


# -*- coding: utf-8 -*-
"""
Created on Sat Oct 24 14:47:05 2020

@author: Balázs Vinkovics
"""

import pickle
import numpy as np
import tensorflow as tf
from tensorflow import keras
import xlwt
from xlwt import Workbook
import os

#Importing dataset
dataset = pickle.load(open("/content/drive/MyDrive/Szakdolgozat/Data/dataset", "rb" ))

#Creating array for storage of internal values
predictions_asd = []
sign = []
grad = []
classifiers = {}

transability = []

forTest = [84]

#Importing classifiers by ids
for index in dataset.keys():
    path = "/content/drive/MyDrive/Szakdolgozat/Classifiers/classifier_id_{}". format(index)
    name = "classifier{}". format(index)
    name = pickle.load(open(path, 'rb'))
    classifiers['{}' .format(index)] = name

with open("/content/drive/MyDrive/Szakdolgozat/SVM_RACE/classifier", 'rb') as pickle_file:
    model_SVM = pickle.load(pickle_file)

# Load the tensorflow model
model = keras.models.load_model('/content/drive/MyDrive/Szakdolgozat/Neural_Network_Race/DNN_race')
model.trainable = False

#Defining loss object
loss_object = tf.keras.losses.CategoricalCrossentropy()

#Defining gradient calculator modell
def create_adversarial_pattern(input_embedding, input_label):
    with tf.GradientTape() as tape:
        tape.watch(input_embedding)
        prediction = model(input_embedding)
        predictions_asd.append(prediction.numpy())
        loss = loss_object(input_label, prediction)

    # Get the gradients of the loss w.r.t to the input image.
    gradient = tape.gradient(loss, input_embedding)
    grad.append(gradient.numpy())
    # Get the sign of the gradients to create the perturbation
    signed_grad = tf.sign(gradient)
    sign.append(signed_grad.numpy())
    return signed_grad

#Choosing datakey (= person)
# ID = 1

#Creating One-Hot encoded input labels
prediction_white = [1, 0, 0, 0]
prediction_black = [0, 1, 0, 0]
prediction_asian = [0, 0, 1, 0]
prediction_indian = [0, 0, 0, 1]

#Declaring epsilon values
epsilons_race = [0.0, 0.005, 0.006, 0.007, 0.008, 0.009, 0.01, 0.012, 0.014]

#Tracking pesron number
person_number = 0

#Creating arrays for data storage
face_embeddings = []
class_error_final = []
class_emb_final = []
class_advex_final = []
prediction_advex_array = []

predSuccessful = {}
classSuccessful = {}

#Creating dict for storage
predSuccessful = {}
for eps in epsilons_race:
    predSuccessful[eps] = []

classSuccessful = {}
for eps in epsilons_race:
    classSuccessful[eps] = []

svmResults = {}

for key in dataset.keys():
    svmResults['{}'. format(key)] = {}

#Iterating through persons
for key in dataset.keys():

    #Creating arrays for data storage, tracking embedding number
    classifier_emb_array = []
    classifier_advex_emb_array = []
    embedding_number = 0

    svmResultsByKey = {}
    for eps in epsilons_race:
        svmResultsByKey['{}'. format(eps)] = []

    input_label_variable = [0, 0, 0, 0]

    if dataset[key]["race"] == "white":
            input_label_variable = prediction_white
            input_label_string = "white"
    if dataset[key]["race"] == "black":
            input_label_variable = prediction_black
            input_label_string = "black"
    if dataset[key]["race"] == "asian":
            input_label_variable = prediction_asian
            input_label_string = "asian"
    if dataset[key]["race"] == "indian":
            input_label_variable = prediction_indian
            input_label_string = "indian"

    #Iterating through the chosen person's embeddings
    for emb in dataset[key]["embeddings"]:
        print("\n\n{}. people" .format(person_number))
        #Converting embedding to tensor
        emb = tf.convert_to_tensor([emb])

        #Defining input label value due to sex (correct label)

        input_label = np.asarray(input_label_variable)
        input_label = np.reshape(input_label, (1, 4))

        #Creating perturbation
        perturbations = create_adversarial_pattern(emb, input_label)
        print("Embedding number: {}" .format(embedding_number))

        #Counting embedding number
        embedding_number = embedding_number + 1

        #Iterating through adv. examples with different perturbations
        for eps in epsilons_race:
            #Creating adversarial example
            advex_emb = eps*perturbations + emb

            #Predicting sex for advex
            predictedRace = int(model.predict_classes(advex_emb))
            prediction_advex_array.append(predictedRace)
            svmPredict = model_SVM.predict(advex_emb)
            svmResultsByKey["{}". format(eps)].append(svmPredict)

            if (predictedRace == 0):
                predictedRace = "white"
            if (predictedRace == 1):
                predictedRace = "black"
            if predictedRace == 2:
                predictedRace = "asian"
            if predictedRace == 3:
                predictedRace = "indian"

            if (svmPredict == 1):
                svmPredict = "white"
            if (svmPredict == 2):
                svmPredict = "black"
            if svmPredict == 3:
                svmPredict = "asian"
            if svmPredict == 4:
                svmPredict = "indian"

            print(eps, ": ", "predicted race for emb:", predictedRace, "ground truth:", dataset[key]["race"], "SVM:    ", svmPredict)

            if(predictedRace != input_label_string):
                if svmPredict != input_label_string:
                    transability.append(1)
                else:
                    transability.append(0)

            #Store face embedding
            face_embeddings.append(advex_emb[0])

            #Identification for original emb and advex
            emb_id = int(classifiers["{}" .format(key)].predict(emb))
            classifier_emb_array.append(emb_id)
            advex_id = int(classifiers["{}" .format(key)].predict(advex_emb))
            classifier_advex_emb_array.append(advex_id)

            if(predictedRace != input_label_string):
                predSuccessful[eps].append(1)

                if(advex_id == 1):
                    classSuccessful[eps].append(1)
                else:
                    classSuccessful[eps].append(0)
            else:
                predSuccessful[eps].append(0)

            print("Classification for emb: ", emb_id, " Classification for advex emb: ", advex_id)

    #Calculating error of identification due advex
    classifier_advex_emb_array = np.asarray(classifier_advex_emb_array)
    classifier_emb_array = np.asarray(classifier_emb_array)
    classification_error_advex = (classifier_advex_emb_array[classifier_advex_emb_array == 1].size) / (classifier_emb_array[classifier_emb_array == 1].size)
    class_error_final.append(classification_error_advex)

    #Storage of classification
    class_emb_final.append(classifier_emb_array)
    class_advex_final.append(classifier_advex_emb_array)

    #Tracking person number
    person_number = person_number + 1

cntForEps = 0

wb = Workbook()
results_excel = wb.add_sheet('Results')
results_excel.write(0, 0, 'Epsilon')
results_excel.write(0, 1, 'Successful classification')
results_excel.write(0, 2, 'Successful identification')
results_excel.write(0, cntForEps+12, 'Epsilon values')
results_excel.write(2, 4, 'SVM')

transability = np.asarray(transability)
SVM_final = (transability[transability == 1].size) / (transability.size)

results_excel.write(3, 4, SVM_final)

for eps in epsilons_race:
    results_excel.write(0, cntForEps+13, epsilons_race[cntForEps])
    cntForEps = cntForEps + 1

lineCounter = 1

# Printing result, for this test not reliable, because divison with 0
for eps in epsilons_race:
    array_ActClass = classSuccessful[eps]
    array_ActClass = np.asarray(array_ActClass)
    print("successfull classicication for eps: ", eps)
    resultOfClassification = array_ActClass[array_ActClass == 1].size / array_ActClass.size
    print(resultOfClassification)
    results_excel.write(lineCounter, 0, eps)
    results_excel.write(lineCounter, 1, resultOfClassification)
    lineCounter = lineCounter  + 1

lineCounter = 1

for eps in epsilons_race:
    array_ActClass = predSuccessful[eps]
    array_ActClass = np.asarray(array_ActClass)
    print("successfull prediction for eps: ", eps)
    resultOfIdentification = array_ActClass[array_ActClass == 1].size / array_ActClass.size
    print(resultOfIdentification)
    results_excel.write(lineCounter, 2, resultOfIdentification)
    lineCounter = lineCounter + 1

with os.scandir('/content/drive/MyDrive/Szakdolgozat/Results/Final/FGSM/Race/') as entries:
    cnt = 1
    for entry in entries:
        cnt = cnt + 1

nameSave = "/content/drive/MyDrive/Szakdolgozat/Results/Final/FGSM/Race/Results_{}". format(cnt) + ".xls"
wb.save(nameSave)


# -*- coding: utf-8 -*-

from odoo import models, fields

class StockPickingType(models.Model):
    _inherit = "stock.picking.type"

    u_allow_swapping_packages = fields.Boolean('Allow swapping packages')
    u_skip_allowed = fields.Boolean(
            string='Skip allowed',
            default=False,
            help='Flag to indicate if the skip button will be shown.',
            )
    u_split_on_drop_off_picked = fields.Boolean('Split on drop off picked')
    u_suggest_qty = fields.Boolean(
        string='Suggest Qty',
        default=True,
        help='If True, suggest quantity on mobile if there is an expected quantity.',
    )
    u_over_receive = fields.Boolean(
        string='Over Receive',
        default=True,
        help='If True, allow additional items not in the ASN, or over the expected quantity, '
             'to be added at goods in.',
    )
    u_enforce_location_dest_id = fields.Boolean(
        string='Enforce Destination Location',
        default=False,
        help='Flag to indicate if the destination location of operations should '
             'be forced to be a child_of the picking location_dest_id.',)
    u_confirm_location_dest_id = fields.Boolean(
        string='Confirm Destination Location',
        default=True,
        help='Flag to indicate whether we need to scan the Destination Location of operations, '
             'or if it is automatically confirmed as the preset Destination Location.',)
    u_display_summary = fields.Selection([
        ('none', 'None'),
        ('list', 'List'),
        ('list_contents', 'List with Contents'),
    ],
        string='Display Summary',
        default='none'
    )
    u_validate_real_time = fields.Boolean(
        string='Validate In Real Time',
        default=False,
        help='When True, operations are automatically validated in real time.'
    )
    u_target_storage_format = fields.Selection([
        ('pallet_products', 'Pallet of products'),
        ('pallet_packages', 'Pallet of packages'),
        ('package', 'Packages'),
        ('product', 'Products'),
    ],
        string='Target Storage Format',
    )
    u_user_scans = fields.Selection([
        ('pallet', 'Pallets'),
        ('package', 'Packages'),
        ('product', 'Products'),],
        string='What the User Scans',
        help='What the user scans when asked to '
        'scan something from pickings of this type')
    u_reserve_as_packages = fields.Boolean(
        string='Reserve entire packages',
        default=False,
        help="Flag to indicate reservations should be rounded up to entire packages."
    )
    u_confirm_serial_numbers = fields.Selection([
        ('no', 'No'),
        ('yes', 'Yes'),
        ('first_last', 'First/Last'),
    ],
        string='Confirm Serial Numbers',
    )

    u_handle_partials = fields.Boolean(
        string='Process Partial Transfers',
        default=True,
        help='Allow processing a transfer when the preceding transfers are not all completed.'
    )

    u_create_procurement_group = fields.Boolean(
        string='Create Procurement Group',
        default=False,
        help='Flag to indicate that a procurement group should be created on '
             'confirmation of the picking if one does not already exist.',
    )

    u_suggest_location = fields.Boolean(
        string='Suggest locations',
        default=False,
        help='Flag to indicate with if picking type should suggest locations '
             'to user'
    )

    def _prepare_info(self):
        """
            Prepares the following extra info of the picking_type in self:
            - u_allow_swapping_packages: boolean
            - u_skip_allowed: boolean
            - u_split_on_drop_off_picked: boolean
            - u_suggest_qty: boolean
            - u_over_receive: boolean
            - u_validate_real_time: boolean
            - u_target_storage_format: string
            - u_user_scans: string
            - u_enforce_location_dest_id: boolean
            - u_reserve_as_packages: boolean
            - u_confirm_serial_numbers: string
        """
        info = super(StockPickingType, self)._prepare_info()
        info.update({
            'u_allow_swapping_packages': self.u_allow_swapping_packages,
            'u_skip_allowed': self.u_skip_allowed,
            'u_split_on_drop_off_picked': self.u_split_on_drop_off_picked,
            'u_suggest_qty': self. u_suggest_qty,
            'u_over_receive': self.u_over_receive,
            'u_validate_real_time': self.u_validate_real_time,
            'u_target_storage_format': self.u_target_storage_format,
            'u_user_scans': self.u_user_scans,
            'u_enforce_location_dest_id': self.u_enforce_location_dest_id,
            'u_confirm_location_dest_id': self.u_confirm_location_dest_id,
            'u_display_summary': self.u_display_summary,
            'u_reserve_as_packages': self.u_reserve_as_packages,
            'u_handle_partials': self.u_handle_partials,
            'u_create_procurement_group': self.u_create_procurement_group,
            'u_confirm_serial_numbers': self.u_confirm_serial_numbers,
            'u_suggest_location': self.u_suggest_location,
            })
        return info


from typing import List
'''
小镇里有 n 个人，按从 1 到 n 的顺序编号。传言称，这些人中有一个暗地里是小镇法官。

如果小镇法官真的存在，那么：

小镇法官不会信任任何人。
每个人（除了小镇法官）都信任这位小镇法官。
只有一个人同时满足属性 1 和属性 2 。
给你一个数组 trust ，其中 trust[i] = [ai, bi] 表示编号为 ai 的人信任编号为 bi 的人。

如果小镇法官存在并且可以确定他的身份，请返回该法官的编号；否则，返回 -1 。

示例 1：

输入：n = 2, trust = [[1,2]]
输出：2
示例 2：

输入：n = 3, trust = [[1,3],[2,3]]
输出：3
示例 3：

输入：n = 3, trust = [[1,3],[2,3],[3,1]]
输出：-1

提示：

1 <= n <= 1000
0 <= trust.length <= 104

来源：力扣（LeetCode）
链接：https://leetcode-cn.com/problems/find-the-town-judge
著作权归领扣网络所有。商业转载请联系官方授权，非商业转载请注明出处。
'''

class Solution:
    def findJudge(self, n: int, trust: List[List[int]]) -> int:
        # 错误题解
        # people = [i[0] for i in trust]
        # judge = list(set([i[1] for i in trust]))
        # is_judge = [i for i in judge if i not in people]
        # if len(is_judge) == 1:
        #     return is_judge[0]
        # else:
        #     return -1
        '''
        解题思路
        法官不相信任何人，
        相信法官的人为 n - 1
        只需要分别统计出，每个人相信了多少，每个人被多少人相信即可，同时满足上述条件的计委法官
        '''
        cnt1 = [0]*(n + 1)
        cnt2 = [0]*(n + 1)
        for x, y in trust:
            cnt1[y] += 1
            cnt2[x] += 1
        for i in range(1, n + 1):
            if cnt1[i] == n - 1 and cnt2[i] == 0:
                return i
        return -1

obj = Solution()
n = 3
trust = [[1, 2], [2, 3]]
print(obj.findJudge(n, trust))

997-找到小镇的法官.py


SITE_NAME = 'Phablytics'

NAV_LINKS = [
    {
        'name': 'Home',
        'path': '/',
    },
    {
        'name': 'Explore',
        'path': '/explore',
    },
    {
        'name': 'Metrics',
        'path': '/metrics',
    },
    {
        'name': 'Reports',
        'path': '/reports',
    },
    {
        'name': 'Users',
        'path': '/users',
    },
    {
        'name': 'Help',
        'path': '/help',
    },
]

# Breadcrumbs are automatically inflected based on the final phrase of a URL path.
# Overrides can be set based on paths in the BREADCRUMB_OVERRIDES dict.
BREADCRUMB_OVERRIDES = {
    '/' : 'Home',
    '/explore': 'Explore',
    '/metrics': 'Metrics',
    '/metrics/alltasks': 'All Tasks',
    '/reports': 'Reports',
    '/users': 'Users',
    '/help': 'Help',
}


from firstfuncs_1618 import *

###### Direct to the velocity data
def load_vel(moornum):
    if moornum==8:
            dat16=xr.open_dataset(datadir+'OSNAP2016recovery/M1_netcdf/M1_vel_2016recovery_hourly.nc')
            dat18=xr.open_dataset(datadir+'OSNAP2018recovery/M1_netcdf/M1_vel_2018recovery_hourly.nc')
    else:
            dat16=xr.open_dataset(datadir+'OSNAP2016recovery/Hourly_netcdf/CF'+str(moornum)+'_vel_2016recovery_hourly_dropadcpbin.nc')
            dat18=xr.open_dataset(datadir+'OSNAP2018recovery/Hourly_netcdf/CF'+str(moornum)+'_vel_2018recovery_hourly_dropadcpbin.nc')

    return dat16,dat18

def load_mcat(moornum):
    if moornum==8:
            dat16=xr.open_dataset(datadir+'OSNAP2016recovery/M1_netcdf/M1_mcat_2016recovery_hourly.nc')
            dat18=xr.open_dataset(datadir+'OSNAP2018recovery/M1_netcdf/M1_mcat_2018recovery_hourly.nc')
    else:
            dat16=xr.open_dataset(datadir+'OSNAP2016recovery/Hourly_netcdf/CF'+str(moornum)+'_mcat_2016recovery_hourly.nc')
            dat18=xr.open_dataset(datadir+'OSNAP2018recovery/Hourly_netcdf/CF'+str(moornum)+'_mcat_2018recovery_hourlymerged.nc')

    return dat16,dat18

##########################################################################################################
################################   Start with vel ########################################################
#########################################################################################################
moornum=4
dat16,dat18=load_vel(moornum)

dat=xr.concat([dat16,dat18],dim='TIME').drop('LATITUDE').drop('LONGITUDE')

# dat=dat.assign_coords(distance=(distvec[moornum-1]))
# for moornum in range(6,9):
#     dat16,dat18=load_vel(moornum)
#     dat_tmp=xr.concat([dat16,dat18],dim='TIME').drop('LATITUDE').drop('LONGITUDE')
#     dat_tmp=dat_tmp.assign_coords(distance=(distvec[moornum-1]))
#     if moornum==8:
#         dat_tmp=dat_tmp.drop('WCUR')
#     dat=xr.concat([dat,dat_tmp],dim='distance')

dat

# dat=dat.transpose('DEPTH','distance','TIME')

# dat['across track velocity']=dat.UCUR*cos(theta)+dat.VCUR*sin(theta)
# dat['along track velocity']=-dat.UCUR*sin(theta)+dat.VCUR*cos(theta)

##########################################################################################################
################################   Add mcat       ########################################################
#########################################################################################################
moornum=4
d16,d18=load_mcat(moornum)

help(resample)
d16_h=d16.resample(TIME='1H').nearest()
d16_h=d16_h.rename({'PRES':'PRES_CTD'})
dat16_h=dat16.resample(TIME='1H').nearest()
dat16_h=dat16_h.rename({'PRES':'PRES_VEL'})
bothdat16=xr.merge([d16_h,dat16_h])
bothdat16

bothdat16.PRES_CTD.plot()
datadir
bothdat16.to_netcdf(datadir+'OSNAP2016recovery/Hourly_netcdf/CF4_mcat_vel_2016recovery_forMonica_200515.nc','w')

d=xr.concat([d16.drop('LATITUDE').drop('LONGITUDE'),d18.drop('LATITUDE').drop('LONGITUDE')],dim='TIME')#
d=d.assign_coords(distance=(distvec[moornum-1]))
for moornum in range(6,9):
    d16,d18=load_mcat(moornum)
    d_tmp=xr.concat([d16.drop('LATITUDE').drop('LONGITUDE'),d18.drop('LATITUDE').drop('LONGITUDE')],dim='TIME')
    d_tmp=d_tmp.assign_coords(distance=(distvec[moornum-1]))
    d=xr.concat([d,d_tmp],dim='distance')
d=d.transpose('DEPTH','distance','TIME')

d['PSAL'][:,2,15:70].plot()
d

d.TIME[15:70]

Dmat=-gsw.z_from_p(d['PRES'].sel(TIME=slice(d.TIME[0],dat.TIME[-1])).values,60)
DVmat=-gsw.z_from_p(dat['PRES'].sel(TIME=slice(d.TIME[0],dat.TIME[-1])).values,60)

datadic={}
datadic['U']=dat['along track velocity'].sel(TIME=slice(d.TIME[0],dat.TIME[-1])).values
datadic['V']=dat['across track velocity'].sel(TIME=slice(d.TIME[0],dat.TIME[-1])).values
datadic['DV']=DVmat
datadic['D']=Dmat
datadic['T']=d['PTMP'].sel(TIME=slice(d.TIME[0],dat.TIME[-1])).values
datadic['S']=d['PSAL'].sel(TIME=slice(d.TIME[0],dat.TIME[-1])).values
datadic['R']=d['PDEN'].sel(TIME=slice(d.TIME[0],dat.TIME[-1])).values

for kk in datadic:
    print(kk)
    print(shape(datadic[kk]))

io.savemat(open(datadir+'OSNAP_CFgridded_2014-2018/hourly/CF_M_hourly_2014-2018_200130.mat','wb'),datadic)


from room import get_beacons

def trelaterate(dists_3_best):
    beacons = get_beacons()
    ap = beacons[dists_3_best[0][0]]
    bp = beacons[dists_3_best[1][0]]
    cp = beacons[dists_3_best[2][0]]

    # dists_3_best = [(0, 2),(1, 3),(2, 2)]
    ra, rb, rc = dists_3_best[0][1], dists_3_best[1][1], dists_3_best[2][1]
    sqr = lambda a: a ** 2
    norm = lambda a: sum(map(lambda b: b ** 2, a)) ** 0.5
    dot = lambda a, b: sum([a[i] * b[i] for i in range(len(a))])
    vector_subtract = lambda a, b: tuple(a[i] - b[i] for i in range(len(a)))
    vector_add = lambda a, b: tuple(a[i] + b[i] for i in range(len(a)))
    vector_divide = lambda a, b: tuple(a[i] / b for i in range(len(a)))
    vector_multiply = lambda a, b: tuple(a[i] * b for i in range(len(a)))

    ex, ey, ez, i, j, d, a, x, y, z, b, p4 = [0] * 12

    ex = vector_divide(vector_subtract(bp, ap), norm(vector_subtract(bp, ap)))

    i = dot(ex, vector_subtract(cp, ap))
    a = vector_subtract(vector_subtract(cp, ap), vector_multiply(ex, i))
    ey = vector_divide(a, norm(a))
    d = norm(vector_subtract(bp, ap))
    j = dot(ey, vector_subtract(cp, ap))

    x = (sqr(ra) - sqr(rb) + sqr(d)) / (2 * d)
    y = (sqr(ra) - sqr(rc) + sqr(i) + sqr(j)) / (2 * j) - (i / j) * x

    b = sqr(ra) - sqr(x) - sqr(y)

    if abs(b) < 0.000001:
        b = 0

    a = vector_add(ap, vector_add(vector_multiply(ex, x), vector_multiply(ey, y)))
    # p4a = vector_add(a, vector_multiply(ez, z))
    # p4b = vector_subtract(a, vector_multiply(ez, z))

    return a


import asyncio
import re

from typing import List, Generator, Any, Union, Dict
from urllib.robotparser import RobotFileParser
from selectolax.parser import HTMLParser

from loaders.response_loader import ResponseLoader, ResponseInformation

class Crawler:
    """
    Asynchronous web crawler.

    seed (str): The starting URL for crawling.
    allowed_domains (List[str]): List of allowed domains to crawl.
    max_depth (int, optional): Maximum crawling depth. Defaults to 6.
    ignore_robots_txt (bool, optional): If True, ignore robots.txt rules. Defaults to False.
    crawl_delay (float, optional): Delay between requests in seconds. Defaults to 1.
    loop (asyncio.AbstractEventLoop, optional): Custom event loop to use. If not provided,
        a new event loop will be created. Defaults to None.
    user_agent (str, optional): User-Agent string for requests. Defaults to "*".
    """

    def __init__(self,
                 seed: str,
                 allowed_domains: List[str],
                 max_depth: int = 6,
                 crawl_delay: float = 1,
                 loop: asyncio.AbstractEventLoop = None,
                 ignore_robots_txt: bool = False,
                 render_pages: bool = False,
                 url_patters: List[str] = None,
                 user_agent: str = "*"):

        self.seed = seed
        self.allowed_domains = allowed_domains
        self.max_depth = max_depth
        self.ignore_robots_txt = ignore_robots_txt
        self.render_pages = render_pages
        self.crawl_delay = crawl_delay
        self.user_agent = user_agent
        self.url_patterns = url_patters

        self._current_depth = 0
        self._to_visit = set()
        self._visited = set()
        self._loop = None
        self._running_tasks = set()

        # robot.txt parser
        self._robot_parser = RobotFileParser()
        self._robot_parser.set_url(self._get_robot_txt_url())
        self._robot_parser.read()

        # debug variables
        self._total_link_build_attempts = 0

        # set event loop
        self._set_event_loop(loop=loop)

    @property
    def has_crawl_delay(self) -> bool:
        """
        Check if there is a crawl delay set.

        Returns:
            bool: True if crawl delay is set, False otherwise.
        """
        return self.crawl_delay > 0

    def start(self):
        """
        Start the crawling process.
        """
        if not self.ignore_robots_txt:
            crawl_delay = self._robot_parser.crawl_delay(self.user_agent)
            self.crawl_delay = crawl_delay if crawl_delay else self.crawl_delay
        self._to_visit.add(self.seed)
        task = self._loop.create_task(self._run())
        self._running_tasks.add(task)

    async def exit(self):
        """
        Print summary statistics on exit.
        """
        await asyncio.gather(*self._running_tasks)
        print("TOTAL SITES VISITED:", len(self._visited))
        print("TOTAL ATTEMPTED LINK BUILDS:", self._total_link_build_attempts)
        print("SITES TO VISIT:", len(self._to_visit))

    def extract_links(self, base_url: str, html: str) -> Generator[str, Any, Any]:
        """
        Extract hrefs from the HTML and try to build URLs from them.

        Args:
            base_url (str): The base URL of the HTML page.
            html (str): The HTML content to extract links from.

        Yields:
            str: The extracted URLs.
        """
        parser = HTMLParser(html)
        for a_tag in parser.css("a"):
            href = a_tag.attributes.get("href")
            self._total_link_build_attempts += 1
            yield ResponseLoader.build_link(base_url, href)

    async def _run(self):
        """
        Internal method to perform the crawling asynchronously.
        """

        new_urls = set()
        while self._to_visit and self._current_depth <= self.max_depth:
            if self.has_crawl_delay:
                # If there's a crawl delay, process URLs one at a time
                # by popping a URL from the _to_visit set
                response_pairs = await ResponseLoader.load_responses(self._to_visit.pop(), render_pages=self.render_pages)
                await asyncio.sleep(self.crawl_delay)
            else:
                # If no crawl delay, process all URLs at once
                response_pairs = await ResponseLoader.load_responses(*self._to_visit, render_pages=self.render_pages)
                self._to_visit.clear()

            for url in self._process_responses(response_pairs):
                if url not in self._visited and self._is_url_allowed(url):
                    self._visited.add(url)
                    new_urls.add(url)

            if not self._to_visit:
                # Once we have processed all the URLs in _to_visit, copy over all the new URLs and increase the depth
                self._to_visit.update(new_urls)
                self._current_depth += 1
                new_urls.clear()

    def _process_responses(self, responses: Dict[str, Union[ResponseInformation, BaseException]]) -> Generator[str, Any, Any]:
        """
        Process the responses and yield extracted URLs.

        Args:
            responses (Dict[str, Union[ResponseInformation, BaseException]]): A dictionary of URLs mapped to responses or exceptions.

        Yields:
            str: Extracted URLs.
        """
        for url, response_info in responses.items():
            self._visited.add(url)
            for new_url in self.extract_links(url, response_info.html):
                yield new_url

    def _set_event_loop(self, loop: asyncio.AbstractEventLoop) -> None:
        """
        Set the event loop, creating a new one if needed.
        """
        if not loop:
            try:
                self._loop = asyncio.get_running_loop()
            except RuntimeError:
                self._loop = asyncio.new_event_loop()
                asyncio.set_event_loop(self._loop)
        else:
            self._loop = loop

    def _get_robot_txt_url(self) -> str:
        """
        Returns the URL to the robot txt file.

        Note:
            If no match is found or an invalid URL is returned, the robot parser's
            method can_fetch will always output the value True.

        Returns:
            str: The URL to the robot.txt file.
        """
        root_url = re.match(r'^https?://([^/]+)', self.seed).group(0)
        return f"{root_url}/robots.txt" if root_url else ""

    def _is_url_allowed(self, url: str) -> bool:
        # if rule patterns are defined and the url does not match any pattern return false
        if self.url_patterns and not any([url_pattern for url_pattern in self.url_patterns if re.search(url_pattern, url)]):
            return False
        if self.ignore_robots_txt:
            return True
        return self._robot_parser.can_fetch(self.user_agent, url)

    def __repr__(self):
        return (
            f"Crawler("
            f"seed='{self.seed}', "
            f"allowed_domains={self.allowed_domains}, "
            f"user_agent='{self.user_agent}', "
            f"crawl_delay={self.crawl_delay}, "
            f"max_crawl_depth={self.max_depth}, "
            f"render_pages={self.render_pages}, "
            f"ignore_robots_txt={self.ignore_robots_txt})"
        )


from __future__ import absolute_import

from opentracing.ext import tags as ext_tags
import re

from ._current_span import current_span_func
from ._singleton import singleton
from .. import utils

try:
    import redis
except ImportError:
    redis = None

# regex to match an ipv4 address
IPV4_RE = re.compile(r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$')

METHOD_NAMES = ['execute_command', 'get', 'set', 'setex', 'setnx']
ORIG_METHODS = {}

@singleton
def install_patches():
    if redis is None:
        return

    def peer_tags(self):
        """Fetch the peer host/port tags for opentracing.

        We do this lazily and cache the result since the host/port won't
        change.
        """
        if not hasattr(self, '_peer_tags'):
            self._peer_tags = []
            conn_info = self.connection_pool.connection_kwargs
            host = conn_info.get('host')
            if host:
                if IPV4_RE.match(host):
                    self._peer_tags.append((ext_tags.PEER_HOST_IPV4, host))
                else:
                    self._peer_tags.append((ext_tags.PEER_HOSTNAME, host))
            port = conn_info.get('port')
            if port:
                self._peer_tags.append((ext_tags.PEER_PORT, port))
        return self._peer_tags

    redis.StrictRedis.peer_tags = peer_tags

    for name in METHOD_NAMES:
        ORIG_METHODS[name] = getattr(redis.StrictRedis, name)

    def get(self, name, **kwargs):
        self._extra_tags = [('redis.key', name)]
        return ORIG_METHODS['get'](self, name, **kwargs)

    def set(self, name, value, ex=None, px=None, nx=False, xx=False, **kwargs):
        self._extra_tags = [('redis.key', name)]
        return ORIG_METHODS['set'](self, name, value,
                                   ex=ex, px=px, nx=nx, xx=xx,
                                   **kwargs)

    def setex(self, name, time, value, **kwargs):
        self._extra_tags = [('redis.key', name),
                            ('redis.ttl', time)]
        return ORIG_METHODS['setex'](self, name, time, value, **kwargs)

    def setnx(self, name, value, **kwargs):
        self._extra_tags = [('redis.key', name)]
        return ORIG_METHODS['setnx'](self, name, value, **kwargs)

    def execute_command(self, cmd, *args, **kwargs):
        operation_name = 'redis:%s' % (cmd,)
        span = utils.start_child_span(
            operation_name=operation_name, parent=current_span_func())
        span.set_tag(ext_tags.SPAN_KIND, ext_tags.SPAN_KIND_RPC_CLIENT)
        span.set_tag(ext_tags.PEER_SERVICE, 'redis')

        # set the peer information (remote host/port)
        for tag_key, tag_val in self.peer_tags():
            span.set_tag(tag_key, tag_val)

        # for certain commands we'll add extra attributes such as the redis key
        for tag_key, tag_val in getattr(self, '_extra_tags', []):
            span.set_tag(tag_key, tag_val)
        self._extra_tags = []

        with span:
            return ORIG_METHODS['execute_command'](self, cmd, *args, **kwargs)

    for name in METHOD_NAMES:
        setattr(redis.StrictRedis, name, locals()[name])

def reset_patches():
    for name in METHOD_NAMES:
        setattr(redis.StrictRedis, name, ORIG_METHODS[name])
    ORIG_METHODS.clear()
    install_patches.reset()


import time
import numpy as np
"""
Ask generator
"""
class gen(object):
    def __init__(self, fc, A=0, fm=0):
        """initialization"""
        if A == 0: return
        from math import pi, cos, sin
        import const as c
        from const import t
        self.fc = fc
        self.fm = fm
        self.A = A / 2
        self.A2 = A * 0.01
        self.A3 = A * 0.001
        omega_fc = 6.28 * fc
        _2omega_fc = 2 * omega_fc
        _3omega_fc = 3 * omega_fc
        if self.fm == 0:
            self.M = 0
            omega_fm = 0
        else:
            self.M = 1
            omega_fm = 6.28 * fm

        _data = [self.A*cos(omega_fc*i)*(1 + self.M * cos(omega_fm *i))+\
        self.A2*cos(_2omega_fc*i)*(1 + self.M * cos(omega_fm *i))+\
        self.A3*cos(_3omega_fc*i)*(1 + self.M * cos(omega_fm *i))\
        for i in t]

        c.inp_signal_buff = [a + b for a, b in zip(c.inp_signal_buff, _data)]
        del _data


import time
from datetime import datetime, timedelta
import backtrader as bt

from ccxtbt import CCXTFeed

def main():
    class TestStrategy(bt.Strategy):
        def __init__(self):
            self.next_runs = 0

        def next(self, dt=None):
            dt = dt or self.datas[0].datetime.datetime(0)
            print('%s closing price: %s' % (dt.isoformat(), self.datas[0].close[0]))
            self.next_runs += 1

    cerebro = bt.Cerebro()

    cerebro.addstrategy(TestStrategy)

    # Add the feed
    #hist_start_date = datetime.utcnow() - timedelta(minutes=120)
    data = CCXTFeed(exchange='binance',
                    dataname='BNB/USDT',
                    timeframe=bt.TimeFrame.Minutes,
                    fromdate=datetime(2021, 8, 1, 0, 0),
                    #todate=datetime(2019, 1, 1, 0, 2),
                    compression=1,
                    ohlcv_limit=1000,
                    drop_newest=True,
                    currency='BNB',
                    retries=5,
                    #debug=True,
                    # 'apiKey' and 'secret' are skipped
                    config={'enableRateLimit': True, 'nonce': lambda: str(int(time.time() * 1000))})
    cerebro.adddata(data)

    # Run the strategy
    cerebro.run()

if __name__ == '__main__':
    main()


import networkx as nx
import matplotlib.pyplot as plt
import os

def plot_dist(edge_list_path):
    G=nx.read_weighted_edgelist(edge_list_path, nodetype=str)
    fig, ax = plt.subplots()
    ax.hist([d for n, d in G.degree()], bins=20, color='skyblue', edgecolor='black')
    ax.set_xlabel('Degree', fontsize=15)
    ax.set_ylabel('Frequency', fontsize=15)
    ax.set_title('Degree Distribution', fontsize=15)
    ax.grid(True)

    base_name = os.path.basename(edge_list_path)
    file_name_without_ext = os.path.splitext(base_name)[0]
    plt.savefig(f"{file_name_without_ext}_dist.png", dpi=1000)


"""added post column

Revision ID: 0de5184abe4b
Revises: c866b38e819f
Create Date: 2021-05-01 12:14:47.337995

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '0de5184abe4b'
down_revision = 'c866b38e819f'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('blogs', sa.Column('post', sa.Text(), nullable=True))
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_column('blogs', 'post')
    # ### end Alembic commands ###


import os
import tempfile
from collections.abc import Generator

from homework09.task01 import merge_sorted_files

def create_temp_files(with_content=[]):
    """Parametrized decorator that creates several temporary files with given content.
    :param with_content: list of byte strings (contain only digits), 1 string for 1 line in file
    :type with_content: List[bytes]
    :return: decorated function that takes temporary files as extra arguments
    :rtype: Callable
    """

    def decorate(func):
        def wrapper(*args, **kwargs):
            files = []
            for i in range(len(with_content)):
                fd, path = tempfile.mkstemp()
                os.write(fd, with_content[i])
                os.close(fd)
                files.append(path)
            try:
                return func(files, *args, **kwargs)
            finally:
                for path in files:
                    if os.path.exists(path):
                        os.unlink(path)

        return wrapper

    return decorate

test_content = [b"1\n4\n7\n8\n", b"2\n3\n", b"5\n6\n9\n", b"10\n", b""]

@create_temp_files(with_content=test_content)
def test_files_merging(temp_files):
    """Test that integerst from files merge in ascending order and function is generator"""
    function_output = merge_sorted_files(temp_files)
    assert list(function_output) == [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
    assert isinstance(function_output, Generator)


import xml.etree.ElementTree as ET
import xml.dom.minidom
import numpy as np
import os

def add_attributes(xml_element, data):
    '''
    Adds attributed to the XML element from a dictionary.
    '''
    for key, val in data.items():
        if isinstance(val, str):
            new_val = val
        elif isinstance(val, tuple):
            new_val = " ".join([str(v) for v in val])
        xml_element.set(key, new_val)

def gen_hole(inner_radius, outer_radius, height, num_facets, fraction=1.0, connect_inner=False, radians=True):
    '''
    Generates a list of box geoms that approximate a hole.
    '''
    # Compute size.
    depth = outer_radius - inner_radius

    # Determine the width based on connecting the inner radius or the outer radius.
    if connect_inner:
        width = 2*np.sqrt((inner_radius/np.cos(np.pi/num_facets))**2 - inner_radius**2)
    else:
        width = 2*np.sqrt((outer_radius/np.cos(np.pi/num_facets))**2 - outer_radius**2)

    # Compute position.
    theta = np.arange(0,num_facets)*2*np.pi/num_facets
    theta *= fraction
    y_pos = np.sin(theta)*(inner_radius + depth/2)
    x_pos = np.cos(theta)*(inner_radius + depth/2)

    # Assemble the geoms.
    geoms = []
    for i in range(num_facets):
        if radians:
            geoms.append({
                'type':'box',
                'pos':(x_pos[i], y_pos[i], height/2),
                'euler':(0, 0, np.pi/2 + theta[i]),
                'size':(width/2, depth/2, height/2),
                'class':'collision'
            })
        else:
            geoms.append({
                'type':'box',
                'pos':(x_pos[i], y_pos[i], height/2),
                'euler':(0, 0, 90+180.0/(np.pi)*theta[i]),
                'size':(width/2, depth/2, height/2),
                'class':'collision'
            })
    geoms.append({
        'type':'cylinder',
        'pos': (0, 0, -height/2),
        'size': (outer_radius, height/2),
        'class': 'collision'
    })
    return geoms

def write_xml(filename, element):
    '''
    Takes an xml.etree.ElementTree element and writes it to a human readable file.
    '''
    # Format and write the XML file.
    xml_string = ET.tostring(element)
    # import pdb; pdb.set_trace()
    xml_dom = xml.dom.minidom.parseString(xml_string)
    pretty_xml_string = xml_dom.toprettyxml()

    os.makedirs(os.path.dirname(filename), exist_ok=True)
    with open(filename, 'w') as f:
        f.write(pretty_xml_string)

def assemble_mujoco_xml(geoms, sites=None, attach_worldbody=False):
    '''
    Assembles a list of geoms into XML that can be read by MuJoCo
    '''
    # Create the XML structure
    data = ET.Element('mujoco')
    if attach_worldbody:
        worldbody = ET.SubElement(data, 'worldbody')
        body = ET.SubElement(worldbody, 'body')
    else:
        body = ET.SubElement(data, 'body')

    # Give the body a name
    add_attributes(body, {'name':'hole'})

    # Adds all of the geoms to the body
    for g in geoms:
        geom_tag = ET.SubElement(body, 'geom')
        add_attributes(geom_tag, g)

    if sites is not None:
        for site in sites:
            site_tag = ET.SubElement(body, 'site')
            add_attributes(site_tag, site)
    return data

if __name__ == "__main__":

    # Generate and save a hole.
    hole_geoms = gen_hole(0.0068, .05, .05, 16)
    hole_sites = [
        {'name':'hole_base', 'pos':(0,0,0), 'euler':(-np.pi,0,0), 'size':(0.005, 0.005, 0.005)},
        {'name':'hole_top', 'pos':(0,0,0.05), 'euler':(-np.pi,0,0), 'size':(0.005, 0.005, 0.005)}
    ]
    hole_xml = assemble_mujoco_xml(hole_geoms, hole_sites)
    filename = 'polyhedral_hole_inner=0-0068_outer=0-05_height=0-05_num_facets=16.xml'
    model_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', 'envs', 'assets', 'hole', filename)
    write_xml(model_path, hole_xml)

    # Generate and save a huge hole.
    hole_geoms = gen_hole(0.04, .05, .05, 16)
    hole_sites = [
        {'name':'hole_base', 'pos':(0,0,0), 'euler':(-np.pi,0,0), 'size':(0.005, 0.005, 0.005)},
        {'name':'hole_top', 'pos':(0,0,0.05), 'euler':(-np.pi,0,0), 'size':(0.005, 0.005, 0.005)}
    ]
    hole_xml = assemble_mujoco_xml(hole_geoms, hole_sites)
    filename = 'polyhedral_hole_inner=0-040_outer=0-05_height=0-05_num_facets=16.xml'
    model_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', 'envs', 'assets', 'hole', filename)
    write_xml(model_path, hole_xml)

    # Generate and save a big hole.
    hole_geoms = gen_hole(0.008, .05, .05, 16)
    hole_sites = [
        {'name':'hole_base', 'pos':(0,0,0), 'euler':(-np.pi,0,0), 'size':(0.005, 0.005, 0.005)},
        {'name':'hole_top', 'pos':(0,0,0.05), 'euler':(-np.pi,0,0), 'size':(0.005, 0.005, 0.005)}
    ]
    hole_xml = assemble_mujoco_xml(hole_geoms, hole_sites)
    filename = 'polyhedral_hole_inner=0-008_outer=0-05_height=0-05_num_facets=16.xml'
    model_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', 'envs', 'assets', 'hole', filename)
    write_xml(model_path, hole_xml)

    # Generate and save a medium hole.
    hole_geoms = gen_hole(0.0075, .05, .05, 16)
    hole_sites = [
        {'name':'hole_base', 'pos':(0,0,0), 'euler':(-np.pi,0,0), 'size':(0.005, 0.005, 0.005)},
        {'name':'hole_top', 'pos':(0,0,0.05), 'euler':(-np.pi,0,0), 'size':(0.005, 0.005, 0.005)}
    ]
    hole_xml = assemble_mujoco_xml(hole_geoms, hole_sites)
    filename = 'polyhedral_hole_inner=0-0075_outer=0-05_height=0-05_num_facets=16.xml'
    model_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', 'envs', 'assets', 'hole', filename)
    write_xml(model_path, hole_xml)

    # Generate and save a small hole.
    hole_geoms = gen_hole(0.0070, .05, .05, 16)
    hole_sites = [
        {'name':'hole_base', 'pos':(0,0,0), 'euler':(-np.pi,0,0), 'size':(0.005, 0.005, 0.005)},
        {'name':'hole_top', 'pos':(0,0,0.05), 'euler':(-np.pi,0,0), 'size':(0.005, 0.005, 0.005)}
    ]
    hole_xml = assemble_mujoco_xml(hole_geoms, hole_sites)
    filename = 'polyhedral_hole_inner=0-0070_outer=0-05_height=0-05_num_facets=16.xml'
    model_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', 'envs', 'assets', 'hole', filename)
    write_xml(model_path, hole_xml)

    # Generate and save a tiny hole.
    hole_geoms = gen_hole(0.00685, .05, .05, 16)
    hole_sites = [
        {'name':'hole_base', 'pos':(0,0,0), 'euler':(-np.pi,0,0), 'size':(0.005, 0.005, 0.005)},
        {'name':'hole_top', 'pos':(0,0,0.05), 'euler':(-np.pi,0,0), 'size':(0.005, 0.005, 0.005)}
    ]
    hole_xml = assemble_mujoco_xml(hole_geoms, hole_sites)
    filename = 'polyhedral_hole_inner=0-00685_outer=0-05_height=0-05_num_facets=16.xml'
    model_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', 'envs', 'assets', 'hole', filename)
    write_xml(model_path, hole_xml)

    # Generate and save many holes
    upper = 0.04
    lower = 0.0068
    minimum = 0.006725
    inner_sizes = np.logspace(np.log10(upper-minimum),np.log10(lower-minimum),100) + minimum

    for i,s in enumerate(inner_sizes):
        hole_geoms = gen_hole(s, .10, .05, 16)
        hole_sites = [
            {'name':'hole_base', 'pos':(0,0,0), 'euler':(-np.pi,0,0), 'size':(0.005, 0.005, 0.005)},
            {'name':'hole_top', 'pos':(0,0,0.05), 'euler':(-np.pi,0,0), 'size':(0.005, 0.005, 0.005)}
        ]
        hole_xml = assemble_mujoco_xml(hole_geoms, hole_sites)
        filename = 'polyhedral_hole_inner=0-{:06d}_outer=0-10_height=0-05_num_facets=16_id={:03d}.xml'.format(int(np.round(s*1000000)), i)
        print(filename)
        model_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', 'envs', 'assets', 'hole', filename)
        write_xml(model_path, hole_xml)


'''Creates an Employee with an Address'''

# Imports
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from database_setup import Base, Employee, Address

engine = create_engine('sqlite:///employees.db')
Base.metadata.bind = engine

DBSession = sessionmaker(bind = engine)
session = DBSession()

# Create Employee
rebeccaEmployee = Employee(name = "Rebecca Allen")
session.add(rebeccaEmployee)
session.commit()

# Create Employee
rebeccaAddress = Address(street = "512 Sycamore Road", zip = "02001", employee = rebeccaEmployee)
session.add(rebeccaAddress)
session.commit()

# Read
employees = session.query(Employee).all()
for employee in employees:
	print(employee.name)

# Update
rebecca = session.query(Employee).filter_by(name = "Rebecca Allen").one()
address = session.query(Address).filter_by(employee_id = rebecca.id).one()
address.street = "281 Summer Circle"
address.zip = "00189"
session.add(address)
session.commit()

# Read
addresses = session.query(Address).all()
for address in addresses:
	print(address.street)
	print(address.zip)


# 删除链表中重复的节点
class ListNode:
    def __init__(self, val):
        self.val = val
        self.next = None
    def createNode(self, nodeP, val):
        curNode = ListNode(val)
        while nodeP.next != None:
            nodeP = nodeP.next
        nodeP.next = curNode
class Solution:
    def deleteSameNode(self, head):
        if head == None or head.next == None:
            return head
        preNode = None
        pNode = head
        while pNode:
            pNext = pNode.next
            needDelete = False
            if pNext and pNode.val == pNext.val:
                needDelete = True
            if not needDelete:
                preNode = pNode
                pNode = pNext
            else:
                toBeDeleted = pNode
                value = pNode.val
                while toBeDeleted and toBeDeleted.val == value:
                    pNode = toBeDeleted.next
                    del toBeDeleted
                    toBeDeleted = pNode
                if preNode == None:
                    head = pNode
                else:
                    preNode.next = pNode
        return head
if __name__ == '__main__':
    s = Solution()
    nodeHead = ListNode(1)
    nodeHead.createNode(nodeHead, 2)
    nodeHead.createNode(nodeHead, 3)
    nodeHead.createNode(nodeHead, 3)
    nodeHead = s.deleteSameNode(nodeHead)


"""
    This module contains the code that perform prediction on a given image or images
"""

import cv2
import argparse
import numpy as np
from pathlib import Path

import torch
import torch.nn as nn
import torch.nn.functional as F

import utils.utils as utils
from utils.img_utils import alpha_overlay, normalize

from data_processing.road_dataset import load_image_, numpy_to_tensor

from models.reknetm1 import RekNetM1
from models.reknetm2 import RekNetM2
from models.lidcamnet_fcn import LidCamNet

from misc.transforms import test_trasformations

def predict(models: nn.ModuleList, img_path, path2save, thresh=0.5):
    """
        Perfrom prediction for single image
        Params:
            models     : NN models
            img_path   : path to an image
            path2save  :
            thresh     : preiction threshold
    """

    img_path = Path(img_path)

    if not img_path.exists():
        raise FileNotFoundError("File '{}' not found.".format(str(img_path)))

    src_img = cv2.imread(str(img_path))

    transform = test_trasformations()
    augmented = transform(image=src_img)
    src_img = augmented["image"]

    img2predict = src_img.copy()
    img2predict = cv2.cvtColor(img2predict, cv2.COLOR_BGR2RGB).astype(dtype=np.float32)
    img2predict = normalize(img2predict)

    img2predict = utils.to_gpu(numpy_to_tensor(img2predict).unsqueeze(0).contiguous()).float()

    if len(models) == 1:
        #evaluate mode
        model = models[0].eval()

        with torch.set_grad_enabled(False):
            predict = model(img2predict)

        #Probs
        predict = F.sigmoid(predict).squeeze(0).squeeze(0)

        mask = (predict > thresh).cpu().numpy().astype(dtype=np.uint8)
        overlayed_img = alpha_overlay(src_img, mask)
    else:
        #Averaging all predictions for one point of test data
        sum_predicts = utils.to_gpu(torch.zeros((1, 1, src_img.shape[0], src_img.shape[1])).float())

        for model in models:
            model.eval()
            with torch.set_grad_enabled(False):
                predict = model(img2predict)
            sum_predicts += F.sigmoid(predict)

        predict = (sum_predicts / len(models)).squeeze(0).squeeze(0).float()

        mask = (predict > thresh).cpu().numpy().astype(dtype=np.uint8)
        overlayed_img = alpha_overlay(src_img, mask)

    #save
    cv2.imwrite(path2save, overlayed_img)

    #show
    cv2.imshow("Predicted", overlayed_img)
    cv2.waitKey(0)
    cv2.destroyAllWindows()

    print("Image '{}' was processed successfully.".format(str(img_path)))

def predict_batch(models: nn.ModuleList, path2images, path2save, thresh=0.5):
    """
        Perfrom prediction for a batch images
        Params:
            models          : NN models
            path2images     : path to an image
            path2save       : should be a dir
            thresh          : preiction threshold
    """

    path2images = Path(path2images)
    path2save = Path(path2save)

    if not path2images.is_dir():
        raise RuntimeError("File '{}' is not dir.".format(str(path2images)))

    if not path2save.is_dir():
        raise RuntimeError("File '{}' is not dir.".format(str(path2save)))

    imgs_paths = sorted(list(path2images.glob("*")))

    count_processed = 0
    for idx, ip in enumerate(imgs_paths):
        src_img = cv2.imread(str(ip))

        transform = test_trasformations()
        augmented = transform(image=src_img)
        src_img = augmented["image"]

        img2predict = src_img.copy()
        img2predict = cv2.cvtColor(img2predict, cv2.COLOR_BGR2RGB).astype(dtype=np.float32)
        img2predict = normalize(img2predict)

        img2predict = utils.to_gpu(numpy_to_tensor(img2predict).unsqueeze(0).contiguous()).float()

        if len(models) == 1:
            model = models[0].eval()

            with torch.set_grad_enabled(False):
                predict = model(img2predict)

            #Probs
            predict = F.sigmoid(predict).squeeze(0).squeeze(0)

            mask = (predict > thresh).cpu().numpy().astype(dtype=np.uint8)
            overlayed_img = alpha_overlay(src_img, mask)
        else:
            #Averaging all predictions for one point of test data
            sum_predicts = utils.to_gpu(torch.zeros((1, 1, src_img.shape[0], src_img.shape[1])).float())

            for model in models:
                model.eval()
                with torch.set_grad_enabled(False):
                    predict = model(img2predict)
                sum_predicts += F.sigmoid(predict)

            predict = (sum_predicts / len(models)).squeeze(0).squeeze(0).float()

            mask = (predict > thresh).cpu().numpy().astype(dtype=np.uint8)
            overlayed_img = alpha_overlay(src_img, mask)

        #save
        cv2.imwrite(str(path2save / "{}".format(ip.name)), overlayed_img)

        print("Image '{}' was processed successfully.".format(str(ip)))
        count_processed += 1

    print("{} images were processed.".format(count_processed))

if __name__ == "__main__":
    parser = argparse.ArgumentParser("Prediction module parameters.")
    parser.add_argument("--mode", type=str, default="single", help="Model of prediction. Can be 'single' of 'multiple'.")
    parser.add_argument("--model-path", type=str, required=True, help="Path to a model or models. If path is a dir then models from this dir will be averaged.")
    parser.add_argument("--model-type", type=str, default="reknetm1")
    parser.add_argument("--path2image", type=str, required=True, help="Path to a single image or dir of images.")
    parser.add_argument("--path2save", type=str, required=True, help="Path to save. Can be a single file or dir.")
    parser.add_argument("--thresh", type=float, default=0.5)

    args = parser.parse_args()

    model_path = Path(args.model_path)

    if args.model_type == "reknetm1":
        model = RekNetM1(num_classes=1,
            ebn_enable=True,
            dbn_enable=True,
            upsample_enable=False,
            act_type="celu",
            init_type="He")
        print("Uses RekNetM1 as the model.")
    elif args.model_type == "reknetm2":
        model = RekNetM2(num_classes=1,
            ebn_enable=True,
            dbn_enable=True,
            act_type="celu",
            upsample_enable=False,
            init_type="He")
        print("Uses RekNetM2 as the model.")
    elif args.model_type == "lcn":
        model = LidCamNet(num_classes=1,
            bn_enable=False)
        print("Uses LidCamNet as the model.")
    else:
        raise ValueError("Unknown model type: {}".format(args.model_type))

    if torch.cuda.is_available():
        model = nn.DataParallel(model, device_ids=None).cuda()

    model_list = nn.ModuleList()
    if model_path.is_file():
        state = torch.load(str(model_path))
        model.load_state_dict(state["model"])
        model_list.append(model)
    else:
        models_paths = sorted(list(model_path.glob("*")))
        for mp in models_paths:
            p = mp / "model.pt"
            state = torch.load(str(p))
            model.load_state_dict(state["model"])
            model_list.append(model)

    if args.mode == "single":
        predict(models=model_list, img_path=args.path2image, path2save=args.path2save, thresh=args.thresh)
    elif args.mode == "multiple":
        predict_batch(models=model_list, path2images=args.path2image, path2save=args.path2save, thresh=args.thresh)
    else:
        raise ValueError("Unknown mode: {}".format(args.mode))


"""squash

Revision ID: c41960c217ac
Revises:
Create Date: 2023-07-06 03:46:54.476229

"""
import litestar
import sqlalchemy as sa
from alembic import op

__all__ = ["downgrade", "upgrade"]

# revision identifiers, used by Alembic.
revision = "c41960c217ac"
down_revision = None
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table(
        "country",
        sa.Column("name", sa.String(), nullable=False),
        sa.Column("population", sa.Integer(), nullable=False),
        sa.Column("id", litestar.contrib.sqlalchemy.types.GUID(length=16), nullable=False),
        sa.Column("_sentinel", sa.Integer(), nullable=True),
        sa.Column("created_at", litestar.contrib.sqlalchemy.types.DateTimeUTC(timezone=True), nullable=False),
        sa.Column("updated_at", litestar.contrib.sqlalchemy.types.DateTimeUTC(timezone=True), nullable=False),
        sa.PrimaryKeyConstraint("id", name=op.f("pk_country")),
    )
    op.create_table(
        "author",
        sa.Column("name", sa.String(), nullable=False),
        sa.Column("dob", sa.Date(), nullable=False),
        sa.Column("country_id", litestar.contrib.sqlalchemy.types.GUID(length=16), nullable=True),
        sa.Column("id", litestar.contrib.sqlalchemy.types.GUID(length=16), nullable=False),
        sa.Column("_sentinel", sa.Integer(), nullable=True),
        sa.Column("created_at", litestar.contrib.sqlalchemy.types.DateTimeUTC(timezone=True), nullable=False),
        sa.Column("updated_at", litestar.contrib.sqlalchemy.types.DateTimeUTC(timezone=True), nullable=False),
        sa.ForeignKeyConstraint(["country_id"], ["country.id"], name=op.f("fk_author_country_id_country")),
        sa.PrimaryKeyConstraint("id", name=op.f("pk_author")),
    )
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table("author")
    op.drop_table("country")
    # ### end Alembic commands ###


from tensorflow.python.framework import config
from tensorflow.python.framework import test_util
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import array_ops_stack
from tensorflow.python.ops import gen_array_ops
from tensorflow.python.ops import manip_ops
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import variables as variables_module
from tensorflow.python.ops.linalg import linalg as linalg_lib
from tensorflow.python.ops.linalg import linear_operator_test_util
from tensorflow.python.platform import test

class _LinearOperatorTriDiagBase(object):

  def build_operator_and_matrix(
      self, build_info, dtype, use_placeholder,
      ensure_self_adjoint_and_pd=False,
      diagonals_format='sequence'):
    shape = list(build_info.shape)

    # Ensure that diagonal has large enough values. If we generate a
    # self adjoint PD matrix, then the diagonal will be dominant guaranteeing
    # positive definitess.
    diag = linear_operator_test_util.random_sign_uniform(
        shape[:-1], minval=4., maxval=6., dtype=dtype)
    # We'll truncate these depending on the format
    subdiag = linear_operator_test_util.random_sign_uniform(
        shape[:-1], minval=1., maxval=2., dtype=dtype)
    if ensure_self_adjoint_and_pd:
      # Abs on complex64 will result in a float32, so we cast back up.
      diag = math_ops.cast(math_ops.abs(diag), dtype=dtype)
      # The first element of subdiag is ignored. We'll add a dummy element
      # to superdiag to pad it.
      superdiag = math_ops.conj(subdiag)
      superdiag = manip_ops.roll(superdiag, shift=-1, axis=-1)
    else:
      superdiag = linear_operator_test_util.random_sign_uniform(
          shape[:-1], minval=1., maxval=2., dtype=dtype)

    matrix_diagonals = array_ops_stack.stack(
        [superdiag, diag, subdiag], axis=-2)
    matrix = gen_array_ops.matrix_diag_v3(
        matrix_diagonals,
        k=(-1, 1),
        num_rows=-1,
        num_cols=-1,
        align='LEFT_RIGHT',
        padding_value=0.)

    if diagonals_format == 'sequence':
      diagonals = [superdiag, diag, subdiag]
    elif diagonals_format == 'compact':
      diagonals = array_ops_stack.stack([superdiag, diag, subdiag], axis=-2)
    elif diagonals_format == 'matrix':
      diagonals = matrix

    lin_op_diagonals = diagonals

    if use_placeholder:
      if diagonals_format == 'sequence':
        lin_op_diagonals = [array_ops.placeholder_with_default(
            d, shape=None) for d in lin_op_diagonals]
      else:
        lin_op_diagonals = array_ops.placeholder_with_default(
            lin_op_diagonals, shape=None)

    operator = linalg_lib.LinearOperatorTridiag(
        diagonals=lin_op_diagonals,
        diagonals_format=diagonals_format,
        is_self_adjoint=True if ensure_self_adjoint_and_pd else None,
        is_positive_definite=True if ensure_self_adjoint_and_pd else None)
    return operator, matrix

  @staticmethod
  def operator_shapes_infos():
    shape_info = linear_operator_test_util.OperatorShapesInfo
    # non-batch operators (n, n) and batch operators.
    return [
        shape_info((3, 3)),
        shape_info((1, 6, 6)),
        shape_info((3, 4, 4)),
        shape_info((2, 1, 3, 3))
    ]

@test_util.with_eager_op_as_function
@test_util.run_all_in_graph_and_eager_modes
class LinearOperatorTriDiagCompactTest(
    _LinearOperatorTriDiagBase,
    linear_operator_test_util.SquareLinearOperatorDerivedClassTest):
  """Most tests done in the base class LinearOperatorDerivedClassTest."""

  def tearDown(self):
    config.enable_tensor_float_32_execution(self.tf32_keep_)

  def setUp(self):
    self.tf32_keep_ = config.tensor_float_32_execution_enabled()
    config.enable_tensor_float_32_execution(False)

  def operator_and_matrix(
      self, build_info, dtype, use_placeholder,
      ensure_self_adjoint_and_pd=False):
    return self.build_operator_and_matrix(
        build_info, dtype, use_placeholder,
        ensure_self_adjoint_and_pd=ensure_self_adjoint_and_pd,
        diagonals_format='compact')

  @test_util.disable_xla('Current implementation does not yet support pivoting')
  def test_tape_safe(self):
    diag = variables_module.Variable([[3., 6., 2.], [2., 4., 2.], [5., 1., 2.]])
    operator = linalg_lib.LinearOperatorTridiag(
        diag, diagonals_format='compact')
    self.check_tape_safe(operator)

  def test_convert_variables_to_tensors(self):
    diag = variables_module.Variable([[3., 6., 2.], [2., 4., 2.], [5., 1., 2.]])
    operator = linalg_lib.LinearOperatorTridiag(
        diag, diagonals_format='compact')
    with self.cached_session() as sess:
      sess.run([diag.initializer])
      self.check_convert_variables_to_tensors(operator)

@test_util.with_eager_op_as_function
@test_util.run_all_in_graph_and_eager_modes
class LinearOperatorTriDiagSequenceTest(
    _LinearOperatorTriDiagBase,
    linear_operator_test_util.SquareLinearOperatorDerivedClassTest):
  """Most tests done in the base class LinearOperatorDerivedClassTest."""

  def tearDown(self):
    config.enable_tensor_float_32_execution(self.tf32_keep_)

  def setUp(self):
    self.tf32_keep_ = config.tensor_float_32_execution_enabled()
    config.enable_tensor_float_32_execution(False)

  def operator_and_matrix(
      self, build_info, dtype, use_placeholder,
      ensure_self_adjoint_and_pd=False):
    return self.build_operator_and_matrix(
        build_info, dtype, use_placeholder,
        ensure_self_adjoint_and_pd=ensure_self_adjoint_and_pd,
        diagonals_format='sequence')

  @test_util.disable_xla('Current implementation does not yet support pivoting')
  def test_tape_safe(self):
    diagonals = [
        variables_module.Variable([3., 6., 2.]),
        variables_module.Variable([2., 4., 2.]),
        variables_module.Variable([5., 1., 2.])]
    operator = linalg_lib.LinearOperatorTridiag(
        diagonals, diagonals_format='sequence')
    # Skip the diagonal part and trace since this only dependent on the
    # middle variable. We test this below.
    self.check_tape_safe(operator, skip_options=['diag_part', 'trace'])

    diagonals = [
        [3., 6., 2.],
        variables_module.Variable([2., 4., 2.]),
        [5., 1., 2.]
    ]
    operator = linalg_lib.LinearOperatorTridiag(
        diagonals, diagonals_format='sequence')

@test_util.with_eager_op_as_function
@test_util.run_all_in_graph_and_eager_modes
class LinearOperatorTriDiagMatrixTest(
    _LinearOperatorTriDiagBase,
    linear_operator_test_util.SquareLinearOperatorDerivedClassTest):
  """Most tests done in the base class LinearOperatorDerivedClassTest."""

  def tearDown(self):
    config.enable_tensor_float_32_execution(self.tf32_keep_)

  def setUp(self):
    self.tf32_keep_ = config.tensor_float_32_execution_enabled()
    config.enable_tensor_float_32_execution(False)

  def operator_and_matrix(
      self, build_info, dtype, use_placeholder,
      ensure_self_adjoint_and_pd=False):
    return self.build_operator_and_matrix(
        build_info, dtype, use_placeholder,
        ensure_self_adjoint_and_pd=ensure_self_adjoint_and_pd,
        diagonals_format='matrix')

  @test_util.disable_xla('Current implementation does not yet support pivoting')
  def test_tape_safe(self):
    matrix = variables_module.Variable([[3., 2., 0.], [1., 6., 4.], [0., 2, 2]])
    operator = linalg_lib.LinearOperatorTridiag(
        matrix, diagonals_format='matrix')
    self.check_tape_safe(operator)

if __name__ == '__main__':
  if not test_util.is_xla_enabled():
    linear_operator_test_util.add_tests(LinearOperatorTriDiagCompactTest)
    linear_operator_test_util.add_tests(LinearOperatorTriDiagSequenceTest)
    linear_operator_test_util.add_tests(LinearOperatorTriDiagMatrixTest)
  test.main()


import pandas as pd
import matplotlib.pyplot as plt
import mpld3
import seaborn as sns
from sklearn.cluster import KMeans

pd.set_option('display.max_columns', 500)
pd.set_option('display.max_rows', 500)
pd.set_option('display.width', 1000)

data = pd.read_csv("all_restaurants_compiled_full.csv")

## total number of restaurants

num_rest = data.groupby(["state"])["cuisine_new"].count()

## number of types of restaurants

g = data.groupby(["state","cuisine_new"])["cuisine_new"].count()
g.sort_index(inplace = True)

gu = g.unstack()

## by cuisine region

filt2 = (data["cuisine_group"] != "other") & (data["cuisine_group"] != "none") & (data["cuisine_group"] != "America, Northern")
data_filt = data.loc[filt2,:]
g2 = data_filt.groupby(["state","cuisine_group"])["cuisine_group"].count()
gs = data_filt.groupby("state")["state"].count()
g3 = g2 / gs

gu2 = g3.unstack()

#colors = ["#00e803","#00be00","#009400","#01eeff","#00aeba","#ff01ee","#ce00c1","#a4009c","#80007c","#560054","#ffe601","#ff8373","#ff2e17","#b40f00","#8e0b00","#5a0600","#ff6f01"]
#plt.figure(figsize=(6,10))

# gu2.plot(kind='barh', stacked=True, figsize=(12,15))#, color = colors)
# plt.gca().invert_yaxis()
# plt.legend(loc='center left', bbox_to_anchor=(1.0, 0.5))
# plt.subplots_adjust(right=0.79)
# plt.savefig("cuisine_by_group")

# plt.show()

# #### MPLD3 TEST

# fig, ax = plt.subplots()

# for c in gu2.columns:
#     print(c)
#     ax.barh(gu2.index.to_list(), gu2[c].fillna(0).to_list(), label = c)
# plt.show()

# mpld3.save_html(fig,"test2.html")

import plotly.graph_objects as go

cols = []
# reorganize data for stacked bar chart
for c in gu2.columns:
    cols.append(go.Bar(name = c,
                       y=gu2.index.to_list(),
                       x=gu2[c].fillna(0).to_list(),
                       orientation='h'))
# create plot
fig = go.Figure(data=cols)
fig.update_layout(barmode='stack', height = 1000)
fig.write_html("restaurants_by_country_region.html")

## subplots by cuisine group

filt3 = (data["cuisine_group"] != "other") & (data["cuisine_group"] != "none") & (data["cuisine_group"] != "America, Northern")
data_filt2 = data.loc[filt3,:]
data_c = data_filt2.set_index(keys=["cuisine_group"])
data_c.sort_index(inplace=True)
groups = data_c.index.value_counts().sort_index()

#fig, ax = plt.subplot(num_groups,1)
fig, ax = plt.subplots(1, 1, figsize=(15,22))
plt.subplots_adjust(top=0.96, bottom=0.08)
plt.grid(b=True, which='both', axis='x', zorder=0)
for n,i in enumerate(groups.index):
    temp = data_c.loc[i,:]
    temp2 = temp.groupby("cuisine_new")["cuisine_new"].count()
    #temp3 = temp2.unstack()
    plt.barh(y=temp2.index, width=temp2, label = i, zorder=3)
    plt.xscale("log")
plt.legend()
plt.gca().invert_yaxis()
plt.title("Cuisines by Country and Region", fontweight="heavy", fontsize=12)
#plt.grid(b=True, which='both', axis='x', zorder=0)
#plt.show()
fig.savefig("cuisines_by_group")

### number of chain restaurants

data_full = pd.read_csv("all_restaurants_compiled_full_with_empty_cuisine.csv")
data_ff = pd.read_csv("all_restaurants_usa_overpass_fast_food_v1.csv")
data_ff.columns = ["id","name","cuisine","lat","lon","state"]
d2 = data_full.append(data_ff)

d2.to_csv("combined_restaurant_and_ff.csv")

d2 = d2[d2["name"] != "none"]
rs = d2.groupby("name")["name"].count().sort_values()
rsc = rs > 3

rm = pd.merge(d2, rsc, left_on="name", right_index=True)
rm.rename(columns={"name_y" : "is_chain"}, inplace=True)

rg = rm.groupby(["state","is_chain"])["is_chain"].count()
rgu = rg.unstack()
#rgu.columns = rgu.columns.droplevel()
rgu["percent_chain"] = rgu[1]/(rgu[1] + rgu[0]) * 100

### most popular chain restaurant

counts = pd.DataFrame(d2.groupby(['state','name'])['name'].count())
counts.rename(columns = {"name" : "count"}, inplace = True)
counts.reset_index()
trans = counts.groupby(['state'])['count'].transform(max)
idx = trans == counts['count']
chains = counts[idx]

# clustering

filt3 = (data["cuisine_new"] != "other") & (data["cuisine_new"] != "none")
data_filt3 = data.loc[filt3,:]
st = data_filt3.groupby("state")["state"].count()
#stb = st > 300

mg = pd.merge(data_filt3,st,left_on = "state", right_index = True)
mg.rename(columns = {"state_y" : "n_restaurants"}, inplace = True)
mg = mg[mg["n_restaurants"] > 300]

mg3 = mg.groupby(["state","cuisine_new"])["cuisine_new"].count()
mg_by_st = mg3 / st * 100
mg3u = mg_by_st.unstack()
mg4 = mg3u.dropna(axis = 'columns')
#data_filt3["st_filt"] = st

#sns.set(style="ticks")
#sns.pairplot(mg4)

kmeans = KMeans(n_clusters=4, random_state=0).fit(mg4)
mg4['class'] = kmeans.labels_
#sns.pairplot(mg4, hue="labels", size = 3)
#sns.set(font_scale = 1.6)
#sns.set(font_scale = 1.2)
sns.pairplot(mg4[["mexican","burger","pizza","bbq","italian","chinese","america","seafood","class"]], hue="class", size = 3)

centers = kmeans.cluster_centers_

fig, ax = plt.subplots(1,1, figsize = (8,5))
plt.subplots_adjust(bottom=0.16,top=0.9)
plt.grid(b=True, which='both', axis='x', zorder=0)
for a,line in enumerate(centers):
    plt.plot(mg4.columns[:19],line,linewidth = 3, label = "Class: " + str(a))
plt.xticks(rotation = 45, fontsize = 12)
plt.yticks(fontsize = 12)
plt.xlabel("Cuisine")
plt.ylabel("Class Center")
plt.title("Clustering Centers",  weight = "heavy")
plt.legend(fontsize = 12)
plt.show()

c_temp = pd.DataFrame(chains).reset_index()
m1 = pd.merge(rgu, c_temp, on = "state", how='left')
m2 = pd.merge(m1, mg4["class"], left_on = "state", right_index = True, how = 'left')

m2.to_csv("food_stats_output.csv")

# cuisine in rural vs metropolitan areas
# create map of cuisine areas and plot number of restaurants as numbers
# bar chart of cuisine types and countries (two levels)
# show distribution of cuisine types by state
# show distribution of american food by state (burger, hotdog, pizza, seafood,)
# mexican/ latin?
# asian
# european
# african
# latin
# variety
# clustering result (base clusters on continents)
# pizza, burger, etc...
# southern
# seafood
# vegetarian/vegan
# look at new england, classify areas


import torch
import random
from torch import nn
import numpy as np
import torchvision
import torchvision.transforms as transforms
import torch.nn.functional as F
from torch.autograd import Function
from torch.utils.data import Dataset, DataLoader
import torch.nn.functional as F
from tqdm import tqdm_notebook
# Siamese
## Utils
def dis(x1,x2):
  return (x1-x2).pow(2).sum(-1).sqrt()

## Loss Function
class ContrastiveLoss(nn.Module):
    """
    Contrastive loss
    Takes embeddings of two samples and a target label == 1 if samples are from the same class and label == 0 otherwise
    """

    def __init__(self, margin):
        super(ContrastiveLoss, self).__init__()
        self.margin = margin
        self.eps = 1e-9

    def forward(self, output1, output2, target, size_average=True):
        distances = (output2 - output1).pow(2).sum(1)#.sqrt()  # squared distances
        losses = 0.5 * (target.float() * distances +
                        (1 + -1 * target).float() *F.relu(self.margin - (distances + self.eps)))#.sqrt()).pow(2))
        return losses.mean() if size_average else losses.sum()

## Model
class SiameseNet(nn.Module):
    def __init__(self, embedding_net):
        super(SiameseNet, self).__init__()
        self.embedding_net = embedding_net

    def forward(self, x1, x2):
        output1 = self.embedding_net(x1)
        output2 = self.embedding_net(x2)
        return output1, output2

    def get_embedding(self, x):
        return self.embedding_net(x)

## Trainer
def train_siamese(epoch, train_loader, model, loss_fn, optimizer, cuda, N, tot, tot_label, rep, m):
    model.train()
    total_loss = 0
    correct = 0.0
    total = 0.0
    bar = tqdm_notebook(enumerate(train_loader))
    for batch_idx, x in bar:
        data = x['data']
        target = x['label'].type(torch.LongTensor)
        if not type(data) in (tuple, list):
            data = (data, )
        if cuda:
            data = tuple(d.cuda() for d in data)
            if target is not None:
                target = target.cuda()
        optimizer.zero_grad()
        outputs = model(*data)

        if type(outputs) not in (tuple, list):
            outputs = (outputs, )

        loss_inputs = outputs
        if target is not None:
            targets = (target, )
            loss_inputs += targets
        loss_outputs = loss_fn(*loss_inputs)
        loss = loss_outputs[0] if type(loss_outputs) in (tuple, list) else loss_outputs
        total_loss += loss.item()
        total += target.size(0)
        loss.backward()
        optimizer.step()
        bar.set_description('Loss: %.3f |(%d/%d)' % (total_loss / (batch_idx + 1.0), 0, total))
        if batch_idx == len(train_loader) - 1:
            x = model.get_embedding(tot)
            x_rep = model.get_embedding(rep)
            dist_matrix = dis(x.unsqueeze(1).expand(x.size(0), N, N), x_rep.unsqueeze(0).expand(x.size(0), N, N))
            d = dist_matrix.min(1)[1] - tot_label
            Acc = d[d == 0].size(0)
            bar.set_description('Loss: %.3f|Acc : %.2f%% (%d/%d)' % (total_loss / (batch_idx + 1), 100. * Acc / len(x), Acc, len(x)))
    return (total_loss / (batch_idx + 1), 100. * Acc / len(x))

def test_siamese(epoch, dataloader_val, model, loss_fn, cuda, N, tot, tot_label, rep, m):
    model.eval()
    test_loss = 0
    total = 0.0
    bar = tqdm_notebook(enumerate(dataloader_val))
    for batch_idx, x in bar:
        data = x['data']
        target = x['label'].type(torch.LongTensor)
        if not type(data) in (tuple, list):
            data = (data, )
        if cuda:
            data = tuple(d.cuda() for d in data)
            if target is not None:
                target = target.cuda()
        outputs = model(*data)
        if type(outputs) not in (tuple, list):
            outputs = (outputs, )

        loss_inputs = outputs
        if target is not None:
            targets = (target, )
            loss_inputs += targets
        loss_outputs = loss_fn(*loss_inputs)
        loss = loss_outputs[0] if type(loss_outputs) in (
            tuple, list) else loss_outputs
        test_loss += loss.data.item()
        total += target.size(0)
        bar.set_description('Loss: %.3f |(%d/%d)' % (test_loss / (batch_idx + 1.0), 0, total))
        if batch_idx == len(dataloader_val) - 1:
            x = model.get_embedding(tot)
            x_rep = model.get_embedding(rep)
            dist_matrix = dis(
                x.unsqueeze(1).expand(x.size(0), N, N),
                x_rep.unsqueeze(0).expand(x.size(0), N, N))
            d = dist_matrix.min(1)[1] - tot_label
            Acc = d[d == 0].size(0)
            bar.set_description('Loss: %.3f|Acc : %.2f%% (%d/%d)' % (test_loss / (batch_idx + 1), 100. * Acc / len(x), Acc, len(x)))
    return (test_loss / (batch_idx + 1), 100. * Acc / len(x))

## Dataloader
class SiameseDataset(Dataset):
    def __init__(self, Data, N):
        self.train = Data.train
        if self.train:
            self.data = Data.data
            self.target = Data.target
            self.label_to_indices = {i: torch.where(self.target == i)[0] for i in range(N)
                                    if len(torch.where(self.target == i)[0]) >= 10}
            x = self.label_to_indices
            pos_pairs = [[x[m][i].item(), x[m][i + 1].item(), 1] for m in x for i in range(-1, len(x[m]) - 1)
                         ] + [[x[m][i].item(), x[m][i - 3].item(), 1] for m in x for i in range(3, len(x[m]) - 1, 3)]
            t = []
            Mm = len(pos_pairs)
            h = Mm // len(x) + 1
            for m in x:
                r1 = torch.full((h, ), m).tolist()
                r2 = random.sample([i for j in range(h // (len(x) - 1) + 1) for i in set(x.keys()) - {m}], h)
                r3 = torch.arange(len(x[m])).tolist()
                if len(r3) < h:
                    r3 += [random.randint(0, len(x[m]) - 1) for i in range(h - len(x[m]))]
                    r3 = random.sample(r3, h)
                else:
                    r3 = random.sample(r3, h)
                r4 = [random.randint(0, len(x[n]) - 1) for n in r2]
                t += torch.tensor([r1, r2, r3, r4]).T.tolist()
            neg_pairs = [[x[m][i], x[n][j], 0] for m, n, i, j in t]
            self.pairs = random.sample(pos_pairs + neg_pairs, len(pos_pairs + neg_pairs))

        else:
            random_state = np.random.RandomState(29)
            self.data = Data.data
            self.target = Data.target
            self.label_to_indices = {i: torch.where(self.target == i)[0] for i in range(N)
                                    if len(torch.where(self.target == i)[0]) >= 10}
            x = self.label_to_indices
            pos_pairs = [[i, random_state.choice(self.label_to_indices[self.target[i].item()]), 1]
                         for i in range(0, len(self.data), 2)]
            t = []
            Mm = len(pos_pairs)
            h = Mm // len(x) + 1
            for m in x:
                r1 = torch.full((h, ), m).tolist()
                r2 = random.sample([i for j in range(h // (len(x) - 1) + 1) for i in set(x.keys()) - {m}], h)
                r3 = torch.arange(len(x[m])).tolist()
                if len(r3) < h:
                    print(h, len(r3))
                    r3 += [random.randint(0, len(x[m]) - 1) for i in range(h - len(x[m]))]
                    r3 = random.sample(r3, h)
                else:
                    r3 = random.sample(r3, h)
                r4 = [random.randint(0, len(x[n]) - 1) for n in r2]
                t += torch.tensor([r1, r2, r3, r4]).T.tolist()
            neg_pairs = [[x[m][i], x[n][j], 0] for m, n, i, j in t]
            self.pairs = random.sample(pos_pairs + neg_pairs, len(pos_pairs + neg_pairs))

    def __len__(self):
        return len(self.pairs)

    def __getitem__(self, idx):
        s1 = torch.tensor(self.data[self.pairs[idx][0]], requires_grad=False)
        s2 = torch.tensor(self.data[self.pairs[idx][1]], requires_grad=False)
        i1 = self.pairs[idx][2]
        return {'data': (s1, s2), 'label': i1}