import os
from django.contrib.auth.views import redirect_to_login
from chat.models import *
from django.db.models.query_utils import Q
from notification.models import *
from user.models import *
from post.models import *
from post.forms import *
from group.models import *
from django.shortcuts import redirect, render
from django.urls import reverse
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.contrib.auth.decorators import login_required
from pydub.silence import split_on_silence
from pydub import AudioSegment
import numpy as np
import librosa
import math
import pickle

def home(request):
    me = None if request.user.id is None else User.objects.get(id=request.user.id)
    if me is None:
        return redirect(reverse('user:login'))

    personnal_chats = ChatBox.objects.filter(Q(user1=me)|Q(user2=me))
    group_chats = list(GroupChatBox.objects.filter(creator=me)) + [join.groupchatbox for join in JoinGroupChat.objects.filter(invitee=me)]

    my_groups = set(list(Group.objects.filter(admins__in=[me])) + list(Group.objects.filter(members__in=[me])))

    #online_users = User.objects.filter(is_online=True)
    online_users = User.objects.filter(Q(is_online=True)&~Q(id=me.id))

    posts = Post.objects.all()

    context = {
        'posts': [{
            'view': 'list',
            'post': post,
            'reactions': Reaction.objects.filter(post=post),
            'comments': Comment.objects.filter(post=post),
        } for post in reversed(posts)],
        'me': me,
        'personnal_chats': [{
            'chat': chat,
            'receiver_id': chat.user2.id if chat.user1 == me else chat.user1.id,
        } for chat in personnal_chats],
        'group_chats': [{
            'chat': chat,
            'latest_msg': GroupMessage.objects.filter(chatbox=chat).order_by('-sent')[0]
        } for chat in group_chats],
        'my_groups': my_groups,
        'online_users': online_users,
        'my_notifications': list(reversed(PostNotification.objects.filter(recipient=me).exclude(actor=me))),
    }
    return render(request, 'home.html', context)

def get_mfcc(file_path):
    y, sr = librosa.load(file_path) # read .wav file
    hop_length = math.floor(sr*0.010) # 10ms hop
    win_length = math.floor(sr*0.025) # 25ms frame
    # mfcc is 12 x T matrix
    mfcc = librosa.feature.mfcc(
        y, sr, n_mfcc=12, n_fft=1024,
        hop_length=hop_length, win_length=win_length)
    # substract mean from mfcc --> normalize mfcc
    mfcc = mfcc - np.mean(mfcc, axis=1).reshape((-1,1))
    # delta feature 1st order and 2nd order
    delta1 = librosa.feature.delta(mfcc, order=1)
    delta2 = librosa.feature.delta(mfcc, order=2)
    # X is 36 x T
    X = np.concatenate([mfcc, delta1, delta2], axis=0) # O^r
    # return T x 36 (transpose of X)
    return X.T # hmmlearn use T x N matrix

def detect_leading_silence(sound, silence_threshold=-42.0, chunk_size=10):
    '''
    sound is a pydub.AudioSegment
    silence_threshold in dB
    chunk_size in ms

    iterate over chunks until you find the first one with sound
    '''
    trim_ms = 0 # ms

    assert chunk_size > 0 # to avoid infinite loop
    while sound[trim_ms:trim_ms+chunk_size].dBFS < silence_threshold and trim_ms < len(sound):
        trim_ms += chunk_size

    return trim_ms

def search(request, filename):
    me = None if request.user.id is None else User.objects.get(id=request.user.id)
    my_groups = set(list(Group.objects.filter(admins__in=[me])) + list(Group.objects.filter(members__in=[me])))

    # Get file audio
    abs_path = "E:/Code/Python/Django/tomo/tomo/voice_search_data/"
    audio_data = AudioSegment.from_file(abs_path+filename, format="wav")
    os.remove(abs_path+filename)

    # split audio into single word's audio
    audio_chunks = split_on_silence(audio_data, min_silence_len=500, silence_thresh=-30)
    # export to folder
    for i, chunk in enumerate(audio_chunks):
        out_file = "tomo/voice_search_data/chunk{0}.wav".format(i)
        print("exporting", out_file)
        chunk.export(out_file, format="wav")

    predict_words = []

    # Predict each segmented audio
    i = 0
    for audio_name in os.listdir('tomo/voice_search_data'):
        if audio_name == 'search.wav':
            continue # ignore if this is the original file
        audio_data = AudioSegment.from_file(abs_path+audio_name, format="wav")

        # trim silence
        start_trim = detect_leading_silence(audio_data)
        end_trim = detect_leading_silence(audio_data.reverse())

        trimmed_sound = audio_data[start_trim:len(audio_data)-end_trim]
        trimmed_sound.export(f"tomo/voice_search_data/trimmed{i}.wav", format="wav")

        # get model
        class_names = ['con', 'học', 'nhà', 'sinh', 'tuyển', 'một', 'hai', 'ba', 'bốn', 'năm', 'sáu', 'bảy', 'tám', 'chín', 'có', 'không', 'ngày', 'tháng', 'lớp']
        model = {}
        for key in class_names:
            name = f"tomo/models/model_{key}.model"
            with open(name, 'rb') as file:
                model[key] = pickle.load(file)

        # predict
        record_mfcc = get_mfcc(f"tomo/voice_search_data/trimmed{i}.wav")
        scores = [model[cname].score(record_mfcc) for cname in class_names]
        predict_word = class_names[np.argmax(scores)]

        # convert word of num into num (if exist)
        '''num = {
            'một': 1,
            'hai': 2,
            'ba': 3,
            'bốn': 4,
            'năm': 5,
            'sáu': 6,
            'bảy': 7,
            'tám': 8,
            'chín': 9,
        }
        if predict_word in num:
            predict_word = num[predict_word]'''

        predict_words.append(predict_word)
        os.remove("tomo/voice_search_data/" + audio_name)
        os.remove(f"tomo/voice_search_data/trimmed{i}.wav")
        i += 1

    # Get posts relating to predicted word
    posts_search_result = []
    all_posts = Post.objects.all()
    for post in all_posts:
        if any(str(predict_word) in post.text for predict_word in predict_words):
            posts_search_result.append(post)

    context = {
        'posts': [{
            'view': 'list',
            'post': post,
            'reactions': Reaction.objects.filter(post=post),
            'comments': Comment.objects.filter(post=post),
        } for post in reversed(posts_search_result)],
        'my_groups': my_groups,
        'predict_words': predict_words,
        'me': me,
    }
    return render(request, 'search_result.html', context)

def conv_to_num(word):
    return {
        'một': 1,
        'hai': 2,
        'ba': 3,
        'bốn': 4,
        'năm': 5,
        'sáu': 6,
        'bảy': 7,
        'tám': 8,
        'chín': 9,
    }[word]


from st7920 import ST7920
from random import randint
from time import sleep
import curses
import collections

SCALE = 4
WIDTH = 128/SCALE
HEIGHT = 64/SCALE

score = 0
alive = True

s = ST7920()

def newfoodpos():
	return [randint(0,WIDTH-1), randint(0,HEIGHT-1)]

def update():
	global headpos, foodpos, score
	if headdir == 0:
		newpos = [headpos[0]+1, headpos[1]]
	elif headdir == 1:
		newpos = [headpos[0], headpos[1]+1]
	elif headdir == 2:
		newpos = [headpos[0]-1, headpos[1]]
	else:
		newpos = [headpos[0], headpos[1]-1]
	if newpos[0]<0: newpos[0] += WIDTH
	if newpos[0]>=WIDTH: newpos[0] = 0
	if newpos[1]<0: newpos[1] += HEIGHT
	if newpos[1]>=HEIGHT: newpos[1] = 0
	if (newpos in snakebits):
		dead()
	if newpos[0]==foodpos[0] and newpos[1]==foodpos[1]:
		foodpos = newfoodpos() # don't remove if we hit the food
		score += 1
	else:
		snakebits.popleft() #remove the last tail bit
	snakebits.append(newpos)
	headpos = newpos
	draw()
	s.redraw()

def dead():
	global alive
	alive = False
	s.clear()
	s.put_text("You died!", ((WIDTH*SCALE)-54)/2, ((HEIGHT*SCALE)/2)-8)
	msg = "Score: " + str(score)
	s.put_text(msg, ((WIDTH*SCALE)-(6*len(msg)))/2, ((HEIGHT*SCALE)/2))
	s.redraw()
	exit()

def draw():
	s.clear()
	s.rect(foodpos[0]*SCALE, foodpos[1]*SCALE, ((foodpos[0]+1)*SCALE)-1, ((foodpos[1]+1)*SCALE)-1)
	for bit in snakebits:
		s.fill_rect(bit[0]*SCALE, bit[1]*SCALE, ((bit[0]+1)*SCALE)-1, ((bit[1]+1)*SCALE)-1)

def showsplash(screen):
	s.clear()
	s.put_text("SNAKE!", ((WIDTH*SCALE)-36)/2, ((HEIGHT*SCALE)/2)-16)
	s.put_text("Arrow keys", ((WIDTH*SCALE)-60)/2, ((HEIGHT*SCALE)/2))
	s.put_text("to control!", ((WIDTH*SCALE)-66)/2, ((HEIGHT*SCALE)/2)+8)
	s.redraw()
	sleep(3)
	while screen.getch() != -1: # clear the input buffer
		pass

def main(screen):
	global headdir
	screen.nodelay(1)

	showsplash(screen)

	while alive:
		char = screen.getch()
		if char==113: exit()
		elif char==curses.KEY_RIGHT and headdir!=2 : headdir = 0
		elif char==curses.KEY_DOWN and headdir!=3: headdir = 1
		elif char==curses.KEY_LEFT and headdir!=0: headdir = 2
		elif char==curses.KEY_UP and headdir!=1: headdir = 3
		update()
		sleep(0.05)
	s.clear()
	s.redraw()

foodpos = newfoodpos()
snakebits = collections.deque()
headpos = [5,5]
snakebits.append([2,5])
snakebits.append([3,5])
snakebits.append([4,5])
snakebits.append(headpos)
headdir = 0 #0:east, 1:south, 2:west, 3:north

curses.wrapper(main)


import numpy as np
import matplotlib.pyplot as plt
import bead_util as bu
import scipy.signal as ss

path = "/data/20180927/bead1/spinning/50s_monitor_5min_gaps"
files = bu.find_all_fnames(path)
index = 0
fdrive = 1210.7
bw = 0.5
bwp = 5.
Ns = 250000
Fs = 5000.
k = 1e-13*(2.*np.pi*370.)**2

df = bu.DataFile()
df.load(files[-2])
df.load_other_data()
df.diagonalize()

drive = df.other_data[2]
resp = ss.detrend(df.pos_data[index])*df.conv_facs[0]/k
drive = ss.detrend(df.other_data[2])*df.conv_facs[0]/k
respft = np.fft.rfft(resp)
driveft = np.fft.rfft(drive)
freqs = np.fft.rfftfreq(Ns, d = 1./Fs)

#plot the data
plt.loglog(freqs, np.abs(respft)*2./Ns)
plt.axvline(x = fdrive, linestyle = '--', color = 'k', label = str(fdrive)+"Hz drive", alpha = 0.5)
plt.legend()
plt.xlabel("Frequency [Hz]")
plt.ylabel("Apparent Displacement [m]")
plt.show()

#plot the zoom

plt.semilogy(freqs, np.abs(respft)*2./Ns)
plt.axvline(x = fdrive, linestyle = '--', color = 'k', label = str(fdrive)+"Hz drive", alpha = 0.5)
plt.legend()
plt.xlabel("Frequency [Hz]")
plt.ylabel("Apparent Displacement [m]")
plt.xlim([fdrive-bwp/2., fdrive+bwp/2.])
plt.show()

#get inst amp and phase

tarr = np.linspace(0., 50., 250000)
respft_line = respft
driveft_line = driveft
respft_line[np.abs(freqs - fdrive)>bw] = 0.
driveft_line[np.abs(freqs - fdrive)>bw] = 0.
anal_signal_resp = ss.hilbert(np.fft.irfft(respft_line))
anal_signal_drive = ss.hilbert(np.fft.irfft(driveft_line))

phir = np.unwrap(np.angle(anal_signal_resp)) - np.unwrap(np.angle(anal_signal_drive))

plt.plot(tarr, np.abs(anal_signal_resp))
plt.xlabel("Time [s]")
plt.ylabel("Instantaneous Amplitude [m]")
plt.ylim([0, 4e-10])
plt.xlim([0, 50])
plt.show()

plt.plot(tarr, np.abs(phir))
plt.xlabel("Time [s]")
plt.ylabel("Drive Response Phase Difference [rad]")
plt.xlim([0, 50])
#plt.ylim([0, 3])
plt.show()


from abc import ABCMeta, abstractmethod
import subprocess
import io
from logging import Logger

class Action(metaclass=ABCMeta):
    def __init__(self, action_id, job, **kwargs):
        self.id = action_id
        self.job = job

    @abstractmethod
    def to_text(self, logger: Logger) -> str:
        pass

class TextAction(Action):
    def __init__(self, action_id, job, **kwargs):
        super(TextAction, self).__init__(action_id, job)
        self.text = kwargs.get("text", "今天又是元气满满的一天")

    def to_text(self, loger: Logger) -> str:
        loger.info(f"Text to text Length: {len(self.text)}")
        return self.text

class CommandAction(Action):
    def __init__(self, action_id, job, **kwargs):
        super(CommandAction, self).__init__(action_id, job)
        self.command = kwargs.get("command", "echo Hello")

    def to_text(self, loger: Logger) -> str:
        proc = subprocess.Popen(self.command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=-1)
        proc.wait()
        stream_stdout = io.TextIOWrapper(proc.stdout, encoding='utf-8')
        stream_stderr = io.TextIOWrapper(proc.stderr, encoding='utf-8')
        str_stdout = str(stream_stdout.read()).strip()
        str_stderr = str(stream_stderr.read()).strip()

        loger.info(f"Command to text Command {self.command} stdout: {str_stdout}")
        loger.info(f"Command to text Command {self.command} stdout: {str_stderr}")

        if len(str_stdout) == 0:
            return str_stdout
        else:
            return str_stdout + "\n" + str_stderr


import time
from pathlib import Path
import torch
import torch.nn as nn
from torch.optim import RMSprop, Adam
from torch.optim.lr_scheduler import ReduceLROnPlateau

from .evaluate import evaluate
from .logger import print_logger

def train_net(net,
              dataloaders,
              device,
              result_path : Path,
              learning_rate: float = 0.1,
              epochs : int = 999,
              ):

    train_loader = dataloaders['Train']
    val_loader = dataloaders['Valid']

    early_stop = 0
    early_stop_criterion = 12
    best_val_score = 0
    total_start_time = time.time()

    logger = print_logger(result_path.joinpath('LOG').with_suffix('.txt'))

    image_path = result_path.joinpath('Prediction')
    image_path.mkdir(exist_ok=True, parents = True)
    checkpoint = result_path.joinpath('Model_Weight').with_suffix('.pth')
    checkpoint.parent.mkdir(exist_ok = True, parents = True)

    optimizer = RMSprop(net.parameters(), lr=learning_rate, weight_decay=1e-8)
    # optimizer = Adam(net.parameters(), lr=learning_rate)
    scheduler = ReduceLROnPlateau(optimizer, mode = 'max', factor = 0.1, patience = 4, min_lr = 1e-5)  # goal: maximize Dice score
    criterion = nn.CrossEntropyLoss()

    for epoch in range(epochs+1):
        start_time = time.time()
        net.train()
        epoch_loss = 0
        for images, true_masks, _ in train_loader :
            images = images.to(device=device, dtype=torch.float32)
            true_masks = true_masks.to(device=device, dtype=torch.float32)

            masks_pred = net(images)
            loss = criterion(masks_pred, true_masks)
            optimizer.zero_grad(set_to_none=True)
            loss.backward()
            optimizer.step()
            epoch_loss += loss.item()
        epoch_loss = epoch_loss / len(train_loader)

        dice_score, sensitivity, specificity = evaluate(net, val_loader, device, image_path)
        scheduler.step(dice_score)

        if dice_score <= best_val_score :
            early_stop += 1
        else :
            early_stop = 0
            best_val_score = dice_score
            torch.save(net.state_dict(), checkpoint)

        if early_stop == early_stop_criterion :
            break

        time_elapsed  = time.time() - start_time
        total_elapsed = time.time() - total_start_time
        total_min = total_elapsed // 60
        total_sec = total_elapsed %  60
        lr = optimizer.param_groups[0]['lr']
        logger(f'[EPOCH : {epoch:3d}/{epochs:3d}] \
| LOSS : [{epoch_loss:.4f}] \
| DICE : [{best_val_score:.4f}] \
| SENSI : [{sensitivity:.4f}] \
| SPECI : [{specificity:.4f}] \
| ES : [{early_stop}/{early_stop_criterion}] \
| LR : [{lr:.5f}] \
| TIME : [{int(time_elapsed):3d}S / {int(total_min):2d}M {int(total_sec):2d}S]'
)

    net.load_state_dict(torch.load(checkpoint))
    final_val_score = evaluate(net, val_loader, device, image_path)
    logger(f'\n\nFINAL VALIDATION SCORE : {final_val_score}')

    return net


import os
import shutil
from object2urdf import ObjectUrdfBuilder
from cleanup_tools import get_immediate_subdirectories
import argparse
import shapenet
from glob import glob
import point_cloud_utils as pcu
import numpy as np
import trimesh

def as_mesh(scene_or_mesh):
    # Utils function that returns a mesh from a trimesh.Trimesh() or trimesh.scene.Scene()
    if isinstance(scene_or_mesh, trimesh.Scene):
        mesh = trimesh.util.concatenate([
            trimesh.Trimesh(vertices=m.vertices, faces=m.faces)
            for m in scene_or_mesh.geometry.values()])
    else:
        mesh = scene_or_mesh
    return mesh

# Update file
def replace_in_file(filepath, original, replacement):
    """Replace original string with replacement string in file at filepath.
    These can be single strings or list of strings."""
    with open(filepath, "r") as file:
        filedata = file.read()

    original = [original] if not isinstance(original, list) else original
    replacement = [replacement] if not isinstance(replacement, list) else replacement
    assert len(original) == len(replacement)

    for idx in range(len(original)):
        filedata = filedata.replace(original[idx], replacement[idx])

    with open(filepath, "w") as file:
        file.write(filedata)

def main(args):
    # Create new directory to place processed files
    new_folder = os.path.join(os.path.dirname(shapenet.__file__), 'ShapeNetCoreV2urdf')
    if not os.path.exists(new_folder):
        os.makedirs(new_folder)

    # Create __init__.py file
    initfile = os.path.join(new_folder, '__init__.py')
    try:
        open(initfile, 'x')
    except FileExistsError:
        pass

    shapenet_folder = os.path.join(os.path.dirname(shapenet.__file__), 'ShapeNetCoreV2')

    subdirs = get_immediate_subdirectories(shapenet_folder)

    for subdir in subdirs:
        category_folder = os.path.join(shapenet_folder, subdir)

        # Create new directory for the ShapeNet category
        new_category_folder = os.path.join(new_folder, subdir)
        if not os.path.exists(new_category_folder):
            os.makedirs(new_category_folder)

        # copy prototype.urdf to subdir
        src_proto = os.path.join(shapenet_folder, '_prototype.urdf')
        dst_proto = os.path.join(new_category_folder, '_prototype.urdf')
        shutil.copy2(src_proto, dst_proto)

        builder = ObjectUrdfBuilder(new_category_folder)

        obj_paths = glob(os.path.join(category_folder, '*', 'models', '*.obj'))

        for obj_path in obj_paths:
            # Create new directory for the ShapeNet object
            new_object_folder = os.path.join(new_category_folder, obj_path.split(os.sep)[-3])
            if not os.path.exists(new_object_folder):
                os.makedirs(new_object_folder)

            if args.watertight:
                # Generate watertight mesh
                mesh = as_mesh(trimesh.load(obj_path))

                if mesh.is_watertight:
                    # Copy .obj to new directory
                    shutil.copy2(obj_path, os.path.join(new_object_folder, 'model.obj'))

                else:
                    vm, fm = pcu.make_mesh_watertight(mesh.vertices, mesh.faces, 50000)
                    watertight_path = os.path.join(new_object_folder, 'model.obj')
                    pcu.save_mesh_vf(watertight_path, vm, fm, dtype=np.float32)

            else:
                # Copy .obj to new directory
                shutil.copy2(obj_path, os.path.join(new_object_folder, 'model.obj'))

            # build urdf
            builder.build_urdf(filename=new_object_folder,
                               force_overwrite=True,
                               decompose_concave=False,
                               force_decompose=False,
                               center=None)

            # rename urdf with their .obj name
            src_urdf_path = glob(os.path.join(new_category_folder, '[!_]*.urdf'))[0]
            dst_urdf_path = os.path.join(new_object_folder, 'model.urdf')
            shutil.move(src_urdf_path, dst_urdf_path)

            # Add flag 'concave=yes' to allow concave meshes in simulators,
            # edit the new urdf with the updated mesh path
            obj_index = dst_urdf_path.split(os.sep)[-2]
            original = [f'filename=\"{obj_index}\"',
                        'collision']
            replacement = ['filename=\"model.obj\"',
                           'collision concave=\"yes\"']
            replace_in_file(dst_urdf_path, original, replacement)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--watertight", default=False, action='store_true', help="Extract watertight meshes and watertight URDF"
    )
    args = parser.parse_args()

    main(args)


from flask import Flask, request
import base64
from PIL import Image
from preprocess4 import Pr1
from preprocess2 import test_transforms
import torch
from torchvision import models
import torch.nn as nn
import numpy as np
import cv2

def get_net():
    finetune_net = nn.Sequential()
    finetune_net.features = models.resnet18(weights='ResNet18_Weights.DEFAULT')

    finetune_net.output_new = nn.Sequential(nn.Linear(1000, 256),
                                            nn.ReLU(),
                                            nn.Linear(256, 26))

    finetune_net = finetune_net.to('cpu')

    for param in finetune_net.features.parameters():
        param.requires_grad = False
    return finetune_net

# Load the saved parameters
saved_params = torch.load('my_model61.pt', map_location=torch.device('cpu'))

# Create a new instance of the model and load the parameters
model_test = get_net()
model_test.load_state_dict(saved_params)
model_test.eval()
classes = ['a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z']

app = Flask(__name__)

@app.route('/upload', methods=['POST'])
def upload():
    try:
        print(0)
        # Get the base64 image string from the request
        base64_image = request.json['image']
        print(1)
        # Decode the base64 image string to bytes
        image_bytes = base64.b64decode(base64_image)
        print(2)

        # Convert the image bytes to a numpy array
        nparr = np.fromstring(image_bytes, np.uint8)

        # Decode the numpy array to an image using OpenCV
        frame = cv2.imdecode(nparr, cv2.IMREAD_UNCHANGED)

        # Process the image as needed
        p1 = Pr1(frame)
        processed_frame = p1.detect_crop_and_segment_hands(p1.image)
        if processed_frame is not None:
                cropped_hand_array = Image.fromarray(processed_frame)
                # Apply the transformations
                #img_tensor = test_transforms(cropped_hand_array)
                #Make a prediction using the model
                #prediction = model_test(img_tensor[None].to("cpu"))
                # Get the predicted label
                #pred_label = classes[torch.max(prediction, dim=1)[1]]
                #print(pred_label)
        # Return a response if needed
        return {'status': 'success'}
    except Exception as e:
        return {'status': 'error', 'message': str(e)}

if __name__ == '__main__':
    app.run()


from typing import Iterable
import torch
import numpy as np
from scipy.spatial.distance import cdist
from tqdm import tqdm
import ot

def cost_matrix(
  data: np.ndarray, cost: str = 'correlation',
  normalize_features: bool = True) -> np.ndarray:
  """Compute an empirical ground cost matrix, i.e. a pairwise distance matrix
  between the rows of the dataset (l1-normalized by default). Accepted
  distances are the ones compatible with Scipy's `cdist`.

  Args:
      data (np.ndarray):
        The input data, samples as columns and features as rows.
      cost (str):
        The metric use. Defaults to `correlation`.
      normalize_features (bool, optional):
        Whether to divide the rows by their sum before
        computing distances. Defaults to True.

  Returns:
      np.ndarray: The pairwise cost matrix.
  """
  if normalize_features:
    sums = data.sum(1).reshape(-1, 1)
    C = cdist(data/sums, data/sums, metric=cost)
  else:
    C = cdist(data, data, metric=cost)
  return C/C.max()

def OT_distance_matrix(
  data: np.ndarray,
  cost: np.ndarray,
  eps: float = .1,
  dtype: torch.dtype = torch.double,
  device: str = 'cuda',
  divide_max: bool = False,
  numItermax: int = 500,
  stopThr: float = 1e-5,
  batch_size: int = 200) -> np.ndarray:
  """Compute the pairwise Optimal Transport distance matrix. We compute
  Sinkhorn Divergences using POT's implementation of the Sinkhorn algorithm.
  Computations are done using PyTorch on a specified device. But the result is
  a numpy array. This allows not saturating the GPU for large matrices.

  Args:
      data (np.ndarray):
        The input data, as a numpy array.
      cost (np.ndarray):
        The ground cost between features.
      eps (float, optional):
        The entropic regularization parameter. Small regularization requires
        more iterations and double precision. Defaults to .1.
      dtype (torch.dtype, optional):
        The torch dtype used for computations. Double is more precise but
        takes up more space. Defaults to torch.double.
      device (str, optional):
        The torch device to compute on, typically 'cpu' or 'cuda'.
        Defaults to 'cuda'.
      divide_max (bool, optional):
        Whether to divide the resulting matrix by its maximum value.
        This can be useful to compare matrices. Defaults to False.
      numItermax (int, optional):
        Used by POT, maximum number of Sinkhorn iterations. Defaults to 500.
      stopThr (float, optional):
        Used by POT, tolerance for early stopping in the Sinkhorn iterations.
        Defaults to 1e-5.
      batch_size (int, optional):
        The batch size, i.e. how many distances can be computed at the same
        time. Should be as large as possible on your hardware. Defaults to 200.

  Returns:
      np.ndarray: The pairwise OT distance matrix.
  """

  # Move the cost to PyTorch.
  C = torch.from_numpy(cost)
  C = C.to(device=device, dtype=dtype)

  # Compute the kernel
  K = torch.exp(-C/eps)

  data_tensor = torch.from_numpy(data)
  data_tensor = data_tensor.to(device=device, dtype=dtype)

  D = torch.zeros(data_tensor.shape[1], data_tensor.shape[1], device='cpu', dtype=dtype)

  pbar = tqdm(total=data_tensor.shape[1]*(data_tensor.shape[1] - 1)//2, leave=False)

  errors = []

  # Iterate over the lines.
  for i in range(data_tensor.shape[1]):
    for ii in np.split(range(i+1), np.arange(batch_size, i+1, batch_size)):

      # Compute the Sinkhorn dual variables
      _, wass_log = ot.sinkhorn(
          data_tensor[:,i].contiguous(), # This is the source histogram.
          data_tensor[:,ii].contiguous(), # These are the target histograms.
          C, # This is the ground cost.
          eps, # This is the regularization parameter.
          log=True, # Return the dual variables
          stopThr=stopThr,
          numItermax=numItermax
      )

      # Compute the exponential dual potentials.
      f, g = eps*wass_log['u'].log(), eps*wass_log['v'].log()

      if len(wass_log['err']) > 0:
        errors.append(wass_log['err'][-1])

      # Compute the Sinkhorn costs.
      # These will be used to compute the Sinkhorn divergences
      wass = (
          f*data_tensor[:,[i]*len(ii)] +
          g*data_tensor[:,ii] -
          eps*wass_log['u']*(K@wass_log['v'])
      ).sum(0)

      # Add them in the distance matrix (including symmetric values).
      D[i,ii] = D[ii,i] = wass.cpu()

      pbar.update(len(ii))

  pbar.close()

  # Get the diagonal terms OT_eps(a, a).
  d = torch.diagonal(D)

  # The Sinkhorn divergence is OT(a, b) - (OT(a, a) + OT(b, b))/2.
  D = D - .5*(d.view(-1, 1) + d.view(1, -1))

  # Make sure there are no negative values.
  assert((D < 0).sum() == 0)

  # Make sure the diagonal is zero.
  D.fill_diagonal_(0)

  if divide_max:
    D /= torch.max(D)

  return D.numpy(), errors

def C_index(D: np.ndarray, clusters: np.ndarray) -> float:
  """Compute the C index, a measure of how well the pairwise distances reflect
  ground truth clusters. Implemented here for reference, but the silhouette
  score (aka Average Silhouette Width) is a more standard metric for this.

  Args:
      D (np.ndarray): The pairwise distances.
      clusters (np.ndarray): The ground truth clusters.

  Returns:
      float: The C index.
  """
  Sw = Nw = 0
  for c in np.unique(clusters):
    idx = np.where(clusters == c)[0]
    Sw += D[idx][:,idx].sum()/2
    Nw += int(len(idx)*(len(idx) - 1)/2)

  els = []
  for i in range(len(D)):
    for j in range(i):
      els.append(D[i, j])
  Smin = np.sort(np.array(els))[:Nw].sum()
  Smax = np.sort(np.array(els))[::-1][:Nw].sum()

  return (Sw - Smin)/(Smax - Smin)


from enum import Enum, auto
from pathlib import Path

import numpy as np
import pandas as pd
import pendulum
import pytest
from whatsappnalysis.lib.custom_types import ChatDataset, Schema
from whatsappnalysis.lib.data_loader import WhatsappLoader

class TestWhatsappLoader:
    """ Tests for ChatDataset """

    test_chat_txt = (
        "2/5/20, 8:38 PM - Author 1: Hello world\n"
        "2/5/20, 8:39 PM - Author 1: I like balloons\n"
        "2/5/20, 8:39 PM - Author 2: I like balloons too!\n"
        "2/5/20, 8:42 PM - Author 3: foo\n"
        "2/5/20, 8:42 PM - Author 3: Balloons are terrible\n"
        "2/5/20, 8:45 PM - Author 2: False\n"
    )

    test_chat_df = pd.DataFrame.from_dict(
        {
            "CHAT_NAME": {
                0: "test_chat",
                1: "test_chat",
                2: "test_chat",
                3: "test_chat",
                4: "test_chat",
                5: "test_chat",
            },
            "TIMESTAMP": {
                0: pendulum.parse("2020-02-05 20:38:00+0000"),
                1: pendulum.parse("2020-02-05 20:39:00+0000"),
                2: pendulum.parse("2020-02-05 20:39:00+0000"),
                3: pendulum.parse("2020-02-05 20:42:00+0000"),
                4: pendulum.parse("2020-02-05 20:42:00+0000"),
                5: pendulum.parse("2020-02-05 20:45:00+0000"),
            },
            "AUTHOR": {
                0: "Author 1",
                1: "Author 1",
                2: "Author 2",
                3: "Author 3",
                4: "Author 3",
                5: "Author 2",
            },
            "MESSAGE": {
                0: "Hello world",
                1: "I like balloons",
                2: "I like balloons too!",
                3: "foo",
                4: "Balloons are terrible",
                5: "False",
            },
            "HAS_MEDIA": {
                0: False,
                1: False,
                2: False,
                3: False,
                4: False,
                5: False,
            },
        }
    )

    class Columns(Enum):
        TIMESTAMP = auto()
        AUTHOR = auto()
        MESSAGE = auto()

    schema = Schema(
        columns=Columns,
        columns_to_dtypes={Columns.TIMESTAMP.name: np.dtype("datetime64[ns]")},
    )

    def test_load_from_txt(self, tmp_path: Path):
        """ Test loading from txt file"""
        # Arrange
        expected = self.test_chat_df.astype({"TIMESTAMP": np.dtype("datetime64[ns]")})
        raw_path = tmp_path / "test_chat.txt"
        with raw_path.open("w") as file:
            file.write(self.test_chat_txt)

        dataset = ChatDataset(schema=self.schema)

        # Act
        result = WhatsappLoader().load_from_txt(raw_path)

        # Assert
        pd.testing.assert_frame_equal(result.data, expected)

    def test_load_from_txt_bad_file(self, tmp_path: Path):
        """ Test loading from txt file"""
        # Arrange
        raw_path = tmp_path / "test_chat.txt"
        with raw_path.open("w") as file:
            file.write("")

        # Act / assert
        with pytest.raises(TypeError):
            WhatsappLoader().load_from_txt(raw_path)


import streamlit as st
# import pandas as pd
import numpy as np
import pydeck as pdk
import plotly.express as px
from ParserXML import *
from ConverterToHTML import *
from VisualTools import *

__all__ = [st, pd, np, pdk, px]
DATE_TIME = "date/time"
local_path = ""
file_name = "Datasets/50k_cleaned_from_xml.csv"

DATA_URL = local_path + file_name

st.title("Motor Vehicle Collisions Analyzer")

st.markdown("This application is a Streamlit dashboard that can \
            be used to analyze XML-file from the State Automobile Inspection ЁЯЪФЁЯТе")
st.markdown("ЁЯФ╡     Author: **Andriy Fedorych**")
st.markdown("ЁЯЯб     GitHub: [**StopFuture**](https://github.com/StopFuture)")

upload_check = False
xml_source_file = st.file_uploader("Upload XML File", type="xml")

if xml_source_file is not None and upload_check is False:
    try:
        context_t = DefParserXML(XMLDictStrategy())
        context_t.strategy = XMLDictStrategy()
        imported = context_t.extract_data(xml_source_file.name)
        Converter = ConverterToHTML(xml_source_file.name)

        @st.cache(persist=True)
        def load_data(imported_data):
            def lowercase(el): return str(el).lower()

            imported_data.rename(lowercase, axis='columns', inplace=True)

            imported_data.dropna(subset=["latitude", "longitude", "injured_persons", "date-time", "on_street_name"],
                                 inplace=True)
            imported_data['date-time'] = pd.to_datetime(imported_data['date-time'], format='%Y-%m-%d %H:%M:%S')
            for name in ["injured_persons", "killed_persons", "injured_pedestrians",
                         "killed_pedestrians", "injured_cyclists", "killed_cyclists", "injured_motorists",
                         "killed_motorists"]:
                imported_data[name] = imported_data[name].astype('int')

            imported_data['latitude'] = imported_data['latitude'].astype('float')
            imported_data['longitude'] = imported_data['longitude'].astype('float')

            return imported_data

        upload_check = True
    except Exception as exp:
        x = exp
        st.markdown("тЪая╕П   я╕П**The file is not from the SAI system, try upload another file**")
else:
    upload_check = False

if upload_check:

    data = load_data(imported)
    origin = data

    st.header("Where are the most people injured in city?")
    injured_people = st.slider("Number of persons injured in vehicle collisions", 0, 18)
    midpoint = (np.average(data["latitude"].dropna(how="any")), np.average(data["longitude"].dropna(how="any")))

    tmp_data = data.query("injured_persons >= @injured_people")[cols]
    HeatMap(data, midpoint, injured_people)

    if st.checkbox("Show Raw Data ", False):
        st.subheader('Raw Data')
        x = (st.text_input("Number of displayed rows : ", value="1"))
        st.write(tmp_data.head(int(x) if x != "" else 0))
    DownloadButton(tmp_data, Converter)

    st.header("How many collisions occur during a given time of day(60 min interval)?")
    hour = st.slider("Hour to look at", 0, 24)
    data = data[data['date-time'].dt.hour == hour]
    st.markdown(f"Vehicle collisions between {hour}:00 and {hour + 1}:00")
    HistMap(data, midpoint)

    if st.checkbox("Show Raw Data", False):
        st.subheader('Raw Data')
        x = (st.text_input("Number of displayed rows: ", value="10"))
        st.write(data.head(int(x) if x != "" else 0))
    st.button(
        f"Extract this data as {Converter.set_source(st.text_input('Select a  name:', value=Converter.source))}.html  ",
        key=None, help=None, on_click=Converter.create_html(tmp_data, Converter.source))

    # Hist
    st.subheader("Breakdown by minute between %i:00 and %i:00" % (hour, (hour + 1) % 24))
    filtered = data[
        (data['date-time'].dt.hour >= hour) & (data['date-time'].dt.hour <= hour + 1)
    ]
    hist = np.histogram(filtered["date-time"].dt.minute, bins=60, range=(0, 60))[0]
    chart_data = pd.DataFrame({'minute': range(0, 60, 1), 'crashes': hist})
    fig = px.bar(chart_data, x="minute", y="crashes", hover_data=["minute", "crashes"], height=500)

    st.write(fig)
    st.markdown("The data may be inaccurate, because most of the time is rounded up to 5 minutes")

    if st.checkbox("Show raw data", False):
        st.subheader('Raw Data')
        st.write(data.head(10))

    st.header("Top dangerous streets by affected class")
    Box(data)

    st.header("Creating html file from source data")
    FinalHtmlCreator(origin, Converter)


# -*- coding: utf-8 -*-

from logbook import Logger

import numpy as np
import pandas as pd

from zipline.data.bundles import register
from zipline.utils.calendars import get_calendar

EXPORT_FOLDER = '/mnt/data/earnings_calls/export/'

log = Logger('zipline_ingest.py')

def bundle_hf_data(price_file, debug = False):

    def ingest(environ,
               asset_db_writer,
               minute_bar_writer,
               daily_bar_writer,
               adjustment_writer,
               calendar,
               cache,
               show_progress,
               output_dir,
               start,
               end):

        log.info("Starting bundle build from %s" % price_file)

        data = pd.read_hdf(price_file)
        data.dropna(subset = ['Open', 'Close'], inplace = True)

        data = data.loc[data.Currency == 'USD']
        data['instrument_key'] = data.instrument_key.str.upper()

        log.info("Importing %d instruments" % len(data.instrument_key.unique()))

        dfMetadata = []

        def read_instruments():
            for sid, (instrument_key, instrument_data) in enumerate(data.groupby('instrument_key')):
                log.debug("Reading instrument %s" % instrument_key)

                log.debug("\tInstrument has %d rows" % len(instrument_data))

                if len(instrument_data) == 0:
                    log.debug("\tNo data for instrument, skipping")
                    continue

                instrument_data.drop_duplicates(subset = ['Date'], inplace = True)

                instrument_data.set_index('Date', inplace = True)
                instrument_data.sort_index(inplace = True)

                #dfData['exchange_open'] = instrument_data.index.map(calendar.is_open_on_minute)
                #dfData = dfData[dfData['exchange_open'] == True]

                start_date = instrument_data.index[0]
                log.debug("\tstart_date %s" % start_date)

                end_date = instrument_data.index[-1]
                log.debug("\tend_date %s" % end_date)

                ac_date = end_date + pd.Timedelta(days=1)
                log.debug("\tac_date %s" % ac_date)

                sessions = get_calendar('NYSE').sessions_in_range(start_date, end_date)
                instrument_data = instrument_data.reindex(sessions)

                # Update our meta data
                dfMetadata.append((sid, instrument_key, start_date, end_date, \
                                        ac_date, instrument_key, "Eikon"))

                instrument_data['High'] = np.nan
                instrument_data['Low'] = np.nan
                instrument_data['Volume'].fillna(1.0, inplace = True)

                instrument_data = instrument_data.loc[:, ['Open', 'High', 'Low', 'Close', 'Volume']]
                instrument_data.columns = ['open', 'high', 'low', 'close', 'volume']
                instrument_data = instrument_data.astype(float)

                yield (sid, instrument_data)

                if debug:
                    break

        liData = read_instruments()

        log.info("calling daily_bar_writer")
        daily_bar_writer.write(liData,
                                show_progress = True)
        log.info("returned from daily_bar_writer")

        dfMetadata = pd.DataFrame(dfMetadata,
                                  columns=['sid', 'asset_name', 'start_date',
                                           'end_date', 'auto_close_date',
                                           'symbol', 'exchange'])\
                        .set_index('sid')

        log.info("calling asset_db_writer")
        log.info(dfMetadata)
        asset_db_writer.write(equities = dfMetadata)
        log.info("returned from asset_db_writer")

        log.info("calling adjustment_writer")
        adjustment_writer.write()
        log.info("returned from adjustment_writer")

    return ingest

register(
    'eikon-data-bundle',
    bundle_hf_data(price_file = EXPORT_FOLDER + "/adjusted_prices.hdf",
                   debug = False),
)


import cv2

img = cv2.imread("sample1.png")
cv2.imwrite("sample2.png", img)
img2 = cv2.imread("sample2.png")
grayImg = cv2.cvtColor(img2, cv2.COLOR_BGR2GRAY)
cv2.imshow("Gray", grayImg)

cv2.waitKey(0)
cv2.destroyAllWindows()


# To add a new cell, type '# %%'
# To add a new markdown cell, type '# %% [markdown]'
# %% [markdown]
# # SciKitLearn 机器学习库
# - VScode中, `ctrl + /` 快速注释代码

# %%
# Sklearn 通用的学习模式
# 案例1. 本例鸢尾花数据集，使用KNN模块实现分类

import numpy as np
from sklearn import datasets
# from sklearn.cross_validation import train_test_split # cross_validation包早已不再使用，功能划入model_selection模块中
from sklearn.model_selection import train_test_split
from sklearn.neighbors import KNeighborsClassifier

iris = datasets.load_iris()     # 加载鸢尾花数据集
iris_X = iris.data              # 属性存入X变量，作为特征向量集合
iris_y = iris.target            # 标签存入y变量，作为目标向量

print(iris_X[:2,:])
print(iris_y)

# %%
# 数据集划分
X_train, X_test, y_train, y_test = train_test_split(iris_X, iris_y, test_size=0.3)
# 将iris_X和iris_y都分别按30%测试集的比例划分train集和test集

# 定义用到的模块
knn = KNeighborsClassifier()    # 使用knn模块训练数据分类
# knn = KNeighborsClassifier(n_neighbors=5)     # K近邻会将邻近点求平均，这里可指定平均邻近几个点的值
knn.fit(X_train, y_train)       # 使用的是fit函数

# 测试预测结果
print(knn.predict(X_test))
print(y_test)

# 可视化（自己增加）
yuc = knn.predict(X_test)   # 预测结果
zhs = y_test                # 实际值
import numpy as np
idx = np.arange(0,len(yuc),1)     # 按元素数（len取值）生成索引，用于x坐标
import matplotlib.pyplot as plt
plt.figure()
plt.scatter(idx,yuc,s=80,c='g',alpha=0.5)   # idx为x，预测和实际值为y
plt.scatter(idx,zhs,s=80,c='r',alpha=0.5)   # 设置图形足够大，颜色区分，有透明度

# %%
# 案例2. 本例波士顿房价数据集，使用linear_model实现线性回归预测

from sklearn import datasets
from sklearn.linear_model import LinearRegression

loaded_data = datasets.load_boston()    # 加载波士顿房价数据集
data_X = loaded_data.data               # 数据的data属性就是特征向量集
data_y = loaded_data.target             # 数据的target属性就是目标函数

model = LinearRegression()      # 使用线性回归模型
model.fit(data_X, data_y)

print(model.predict(data_X))
print(data_y)

# 可视化（自己增加）
yuc = model.predict(data_X)     # 预测结果
zhs = data_y                    # 实际值
import numpy as np
idx = np.arange(0,len(yuc),1)       # 按元素数（len取值）成索引，用于x坐标
import matplotlib.pyplot as plt
plt.figure(figsize=(12,4))
plt.plot(idx,yuc,c='g',alpha=0.5)   # idx为x，预测和实际值为y
plt.plot(idx,zhs,c='r',alpha=0.5)   # 设置颜色区分，有透明度

# %%
# model模块的常见属性和功能，如上述的predict预测功能（1分类2回归）

model = LinearRegression()  # 指定本例所用的model
model.fit(X,y)      # 对特征向量集和目标向量，用模型进行拟合
model.predict(X)    # 对测试集数据X,用模型进行预测

model.coef_         # 模型的斜率
model.intercept_    # 模型的截距
model.get_params()  # 获得模型选择时给模型定义的参数
model.score(X,y)    # 对预测结果打分。用X预测，用y做真值进行比较。R^2方式打分

# %%
# 预处理preprocessing
# 标准化normalization、正则化、特征缩放feature scaling
# Idea: Make sure features are on a similar scale. 各特征处于相近的量级，便于学习

from sklearn import preprocessing
X = preprocessing.scale(X)  # 对数据进行预处理（标准化，缩放到0-1之间的数值）

# %%
# 交叉验证（数据集分割）

# 上面案例1中的数据集分割方式，按照固定比例分割
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3)

# 为了有效评价模型，对数据集进行多次不同模式的分割，分别测试并平均其准确率
from sklearn.model_selection import cross_val_score     # cross_val_score函数也并入model_selection
knn = KNeighborsClassifier(n_neighbors=5)               # 计算5个近邻点
score = cross_val_score(knn, X, y, cv=5, scoring='accuracy')    # 分类问题用准确率
# 打分由多次分割评估结果平均而来，使用knn模型，对X预测，用y验证，使用5种分割方案，打分使用准确率进行
loss = -cross_val_score(knn, X, y, cv=5, scoring='neg_mean_squared_error')  # 回归问题用均方差（原值时负值）
# 原mean_squared_error参数已弃用

# %%
# 学习率曲线，可视化学习的准确率变化过程
from sklearn.model_selection import learning_curve      # 学习曲线也放入model_selection
from sklearn.datasets import load_digits
from sklearn.svm import SVC
import matplotlib.pyplot as plt
import numpy as np

digits = load_digits()      # 加载数据集
X = digits.data             # digits属性作为特征向量集
y = digits.target           # 目标向量

# 学习曲线计算（指定阶段的准确率/损失值变化），输出给训练集大小、训练集损失、测试集损失等变量
# gamma是学习率（速率），阶段有数组指定，损失计算和上述交叉验证方法一样
train_sizes, train_loss, test_loss = learning_curve(
        SVC(gamma=0.001),X,y,cv=10,scoring='neg_mean_squared_error',
        train_sizes=[0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1])
train_loss_mean = -np.mean(train_loss,axis=1)   # 上述cv10次分割的值求均值
test_loss_mean = -np.mean(test_loss,axis=1)

plt.plot(train_sizes, train_loss_mean, 'o-',color='r', label="training")
plt.plot(train_sizes, test_loss_mean, 'o-',color='g', label="cross-validation")
plt.xlabel('training examples')
plt.ylabel('loss')
plt.legend(loc='best')
plt.show()

# %%
# 模型调参过程，使用validation_curve评估参数取值变化过程中评估指标的变化曲线，根据是否欠拟合或过拟合来选取该参数的合适范围

from sklearn.model_selection import validation_curve    # 评估曲线也放入model_selection
from sklearn.datasets import load_digits
from sklearn.svm import SVC
import matplotlib.pyplot as plt
import numpy as np

digits = load_digits()
X = digits.data
y = digits.target

param_range = np.logspace(-6, -2.3, 10)          # 在区间取5个点，用于测试参数（调参）
# 评估曲线计算（指定阶段的准确率/损失值变化），输出给训练集大小、训练集损失、测试集损失等变量
# gamma是学习率（速率），阶段有数组指定，损失计算和上述交叉验证方法一样
train_loss, test_loss = validation_curve(       # 改用评估曲线，返回值没有train_sizes
                                                # SVC的固定参数去掉，后面给出参数名和取值范围（已指定）
        SVC(),X,y,param_name='gamma',param_range=param_range, cv=10,scoring='neg_mean_squared_error')
train_loss_mean = -np.mean(train_loss,axis=1)   # 上述cv10次分割的值求均值
test_loss_mean = -np.mean(test_loss,axis=1)

plt.plot(param_range, train_loss_mean, 'o-',color='r', label="training")
plt.plot(param_range, test_loss_mean, 'o-',color='g', label="cross-validation")
plt.xlabel('gamma')
plt.ylabel('loss')
plt.legend(loc='best')
plt.show()

# %%
# 保存model和参数
# pickle方法
import pickle
with open('/path/to/file.pickle','wb') as f:    # 打开句柄-写入
    pickle.dump(model,f)        # 保存模型
with open('/path/to/file.pickle','rb') as f:    # 打开句柄-读出
    mdl = pickle.load(f)        # 加载模型
    print(mdl.predict(X[0:1]))  # 使用模型预测

# joblib方法-sklearn
from sklearn.externals import joblib
joblib.dump(model,'/path/to/file.pkl')  # 保存模型
mdl = joblib.load('/path/to/file.pkl')  # 加载模型
print(mdl.predict(X[0:1]))              # 使用模型预测


import yaml

class Config:
    def __init__(self):
        self.load_model_epochs = None
        self.debug = None
        self.n_epochs = None
        self.load_g_model_score = None
        self.load_d_model_score = None
        self.model_no = None
        self.batch_size = None
        self.n_split = None
        self.max_lr = None
        self.min_lr = None
        self.lambda1 = None
        self.lambda2 = None
        self.seed = None
        self.dataloader_seed = None
        self.device = None
        self.size = None
        self.load()

    def load(self):
        with open('config/config.yml', 'r') as f:
            config = yaml.load(f, Loader=yaml.SafeLoader)
            self.load_model_epochs = config.get('LOAD_MODEL_EPOCH')
            self.debug = config.get('DEBUG')
            self.n_epochs = config.get('N_EPOCHS')
            self.load_g_model_score = config.get('LOAD_G_MODEL_SCORE')
            self.load_d_model_score = config.get('LOAD_D_MODEL_SCORE')
            self.model_no = config.get('MODEL_NO')
            self.batch_size = config.get('BATCH_SIZE')
            self.n_split = config.get('N_SPLIT')
            self.max_lr = config.get('MAX_LR')
            self.min_lr = config.get('MIN_LR')
            self.lambda1 = config.get('LAMBDA1')
            self.lambda2 = config.get('LAMBDA2')
            self.seed = config.get('SEED')
            self.dataloader_seed = config.get('DATALOADER_SEED')
            self.device = config.get('DEVICE')
            self.size = config.get('SIZE')

class TestConfig:
    def __init__(self):
        self.load_model_epochs = None
        self.debug = None
        self.load_g_model_score = None
        self.batch_size = None
        self.seed = None
        self.device = None
        self.size = None
        self.load()

    def load(self):
        with open('config/test_config.yml', 'r') as f:
            config = yaml.load(f, Loader=yaml.SafeLoader)
            self.load_model_epochs = config.get('LOAD_MODEL_EPOCH')
            self.debug = config.get('DEBUG')
            self.load_g_model_score = config.get('LOAD_G_MODEL_SCORE')
            self.batch_size = config.get('BATCH_SIZE')
            self.seed = config.get('SEED')
            self.device = config.get('DEVICE')
            self.size = config.get('SIZE')


from pynvml import *
import logging

from datasets import load_dataset
from datasets import ClassLabel
from transformers import LukeTokenizer, LukeModel, LukeForEntityPairClassification, TrainingArguments, Trainer
import torch
from tqdm import trange
# construir função que converta spans de relativos a frase para globais
import random
import os
import json

def print_gpu_utilization():
    nvmlInit()
    handle = nvmlDeviceGetHandleByIndex(0)
    info = nvmlDeviceGetMemoryInfo(handle)
    print(f"GPU memory occupied: {info.used//1024**2} MB.")

class MyDataset(torch.utils.data.Dataset):
    def __init__(self, encodings, labels):
        self.encodings = encodings
        self.labels = labels

    def __getitem__(self, idx):
        item = {key: val[idx].clone().detach() for key, val in self.encodings.items()}
        item['labels'] = torch.tensor(self.labels[idx])
        return item

    def __len__(self):
        return len(self.labels)

def convert_spans(item):
    sents = []
    sent_map = []

    entities = item["vertexSet"]
    entity_start, entity_end = [], []
    mention_types = []
    entity_spans = []
    for entity in entities:
        for mention in entity:
            if mention["sent_id"] != 0:
                current_id = mention["sent_id"]
                mention["pos"] = [sum(len(s) for s in item["sents"][:current_id])+mention["pos"][0],
                    sum(len(s) for s in item["sents"][:current_id])+mention["pos"][1]]
                mention["sent_id"] = 0

            pos = mention["pos"]
            mention_types.append(mention['type'])
            entity_spans.append(pos)
    item["vertexSet"] = entities
    return item, entity_spans

def load_examples_test(dataset):
    examples = []

    for i, item in enumerate(dataset["validation"]):
        concat_tokens = []
        counter = 0
        converted_item, entity_spans = convert_spans(item)
        tokens = item["sents"]
        for j in range(len(tokens)):
            concat_tokens += tokens[j]
        del j
        tokens = concat_tokens
        del concat_tokens

        # new
        text = ""
        cur = 0
        new_char_spans = [0]*len(entity_spans)
        entity_spans.sort(key=lambda y:y[0])
        for target_entity in entity_spans:
            tamanho_texto = len(text)
            text += " ".join(tokens[cur: target_entity[0]])
            if text:
                text += " "
            char_start = len(text)
            text += " ".join(tokens[target_entity[0]: target_entity[1]])
            char_end = len(text)
            new_char_spans[counter] = (char_start, char_end)
            text += " "
            cur = target_entity[1]
            counter+=1
        text += " ".join(tokens[cur:])
        text = text.rstrip()
        # get true labels
        labels_pairs = tuple(zip(item["labels"]["head"], item["labels"]["tail"], item["labels"]["relation_id"]))
        entity_spans = [tuple(l) for l in entity_spans]
        oldToNewPos =  dict(zip(entity_spans, new_char_spans))
        entities = item["vertexSet"]
        correlations = []
        for pair in labels_pairs:
            for head in entities[pair[0]]:
                if tuple(head["pos"]) in oldToNewPos:
                    head["pos"]=oldToNewPos[tuple(head["pos"])]
                for tail in entities[pair[1]]:
                    if tuple(tail["pos"]) in oldToNewPos:
                        tail["pos"] = oldToNewPos[tuple(tail["pos"])]
                    pack = tuple((head["pos"], tail["pos"], pair[2]))
                    correlations += (pack),
            item["vertexSet"] = entities
            examples.append(dict(
                text=text,
                entity_spans= [d[:][:-1] for d in correlations],
                labels = [d[:][-1] for d in correlations]
            ))
        return examples

def load_examples_competition(dataset):
    examples = []

    for i, item in enumerate(dataset["test"]):
        concat_tokens = []
        counter = 0
        converted_item, entity_spans = convert_spans(item)
        tokens = item["sents"]
        for j in range(len(tokens)):
            concat_tokens += tokens[j]
        del j
        tokens = concat_tokens
        del concat_tokens

        # new
        text = ""
        cur = 0
        new_char_spans = [0]*len(entity_spans)
        entity_spans.sort(key=lambda y:y[0])
        for target_entity in entity_spans:
            tamanho_texto = len(text)
            text += " ".join(tokens[cur: target_entity[0]])
            if text:
                text += " "
            char_start = len(text)
            text += " ".join(tokens[target_entity[0]: target_entity[1]])
            char_end = len(text)
            new_char_spans[counter] = (char_start, char_end)
            text += " "
            cur = target_entity[1]
            counter+=1
        text += " ".join(tokens[cur:])
        text = text.rstrip()
        aux_head = 0
        aux_tail = 0

        labels_pairs = []
        # get true labels
        for head_id in range(len(item["vertexSet"])):
            for tail_id in range(len(item["vertexSet"])):
                if (head_id!=tail_id):
                    labels_pair = tuple([head_id, tail_id , "Na"])
                    labels_pairs.append(labels_pair)

        entity_spans = [tuple(l) for l in entity_spans]
        oldToNewPos =  dict(zip(entity_spans, new_char_spans))
        entities = item["vertexSet"]
        correlations = []
        for pair in labels_pairs:
            head = random.choice(entities[pair[0]])
            tail = random.choice(entities[pair[1]])
            entity_head_id = pair[0]
            entity_tail_id = pair[1]
            rel = pair[2]

            if tuple(head["pos"]) in oldToNewPos:
                head["pos"]=oldToNewPos[tuple(head["pos"])]
            if tuple(tail["pos"]) in oldToNewPos:
                tail["pos"] = oldToNewPos[tuple(tail["pos"])]
            pack = tuple((head["pos"], tail["pos"], pair[2], tuple([entity_head_id, entity_tail_id]), item["title"]))

            item["vertexSet"] = entities
            examples.append(dict(
                text=text,
                entity_spans= pack[:2],
                labels = pack[2],
                idxs_entity_pair = pack[3],
                title = pack[4]
            ))
    return examples

torch.cuda.empty_cache()

dataset = load_dataset("docred")
max_value = 0
#for i, item in enumerate(dataset["train_annotated"]):
#    total_text_len = 0
#    tokens = item["sents"]
#    num_relations = len(item["labels"]["head"])

class ModifiedClassicLuke(LukeForEntityPairClassification):
    def __init__(self, config):
        super().__init__(config)
        self.classifier = torch.nn.Linear(in_features = 2048, out_features = 97, bias = True)

logging.info("Loading data and finetuned dataset for CLASSIC LUKE")
# FAZER LOAD DO MODEL FINETUNED DE 3 EPOCHS
model = ModifiedClassicLuke.from_pretrained("model_finetuned_classic")
tokenizer = LukeTokenizer.from_pretrained("studio-ousia/luke-large-finetuned-tacred")

test_examples = load_examples_competition(dataset)
maximum = 0
max_seq = 0

logging.info("Memory before choosing GPU")
#torch.cuda.empty_cache()

########################## Choose GPU ########################
# set the GPU device to use
cuda_device= 0 # mudar para 0 para dar o cuda
if cuda_device < 0:
    device = torch.device("cpu")
else:
    device = torch.device(f"cuda:{cuda_device}")
#model = model.to(device)
#model.eval()

# Convert to inputs
for batch_start_idx in trange(0, len(test_examples), len(test_examples)):
    batch_examples = test_examples[batch_start_idx:batch_start_idx+len(test_examples)]
    texts = [example["text"] for example in batch_examples]
    entity_spans = [example["entity_spans"] for example in batch_examples]
    #gold_labels = [example["labels"] for example in batch_examples]
    idxs_entity_pair = [example["idxs_entity_pair"] for example in batch_examples]
    titles = [example["title"] for example in batch_examples]

for i in range(len(entity_spans)):
    entity_spans[i] = list(entity_spans[i])

del batch_examples

logging.info("Removing too big examples!!")
num_rejected = 0
clean_texts = []
clean_ents = []
clean_idxs_entity_pairs = []
clean_titles = []
tokenizer2 = LukeTokenizer.from_pretrained("studio-ousia/luke-large")
for ix in range(len(texts)):
    input = tokenizer2(texts[ix])
    if len(input.data["input_ids"]) > 500:
        num_rejected+=1
        continue
    clean_texts.append(texts[i])
    clean_ents.append(entity_spans[ix])
    clean_idxs_entity_pairs.append(idxs_entity_pair)
    clean_titles.append(titles)
texts = clean_texts
entity_spans = clean_ents
idxs_entity_pair = clean_idxs_entity_pairs
titles = clean_titles

torch.cuda.empty_cache()

relations_code_list = ["P1376",
                "P607",
                "P136",
                "P137",
                "P131",
                "P527",
                "P1412",
                "P206",
                "P205",
                "P449",
                "P127",
                "P123",
                "P86",
                "P840",
                "P355",
                "P737",
                "P740",
                "P190",
                "P576",
                "P749",
                "P112",
                "P118",
                "P17",
                "P19",
                "P3373",
                "P6",
                "P276",
                "P1001",
                "P580",
                "P582",
                "P585",
                "P463",
                "P676",
                "P674",
                "P264",
                "P108",
                "P102",
                "P25",
                "P27",
                "P26",
                "P20",
                "P22",
                "Na",
                "P807",
                "P800",
                "P279",
                "P1336",
                "P577",
                "P570",
                "P571",
                "P178",
                "P179",
                "P272",
                "P170",
                "P171",
                "P172",
                "P175",
                "P176",
                "P39",
                "P30",
                "P31",
                "P36",
                "P37",
                "P35",
                "P400",
                "P403",
                "P361",
                "P364",
                "P569",
                "P710",
                "P1344",
                "P488",
                "P241",
                "P162",
                "P161",
                "P166",
                "P40",
                "P1441",
                "P156",
                "P155",
                "P150",
                "P551",
                "P706",
                "P159",
                "P495",
                "P58",
                "P194",
                "P54",
                "P57",
                "P50",
                "P1366",
            "P1365",
            "P937",
            "P140",
            "P69",
            "P1198",
            "P1056"]

c2l = ClassLabel(num_classes = 97, names = relations_code_list)
label_list_ids = [c2l.str2int(label) for label in relations_code_list]
#gold_labels_ids = [c2l.str2int(label) for label in gold_labels]
#aa = [c2l.int2str(label) for label in gold_labels_ids] # convert ints to CODE of label!! USE IN EVAL

#inputs = tokenizer(text=texts[0], entity_spans = entity_spans[0], padding = "max_length", max_length = 1024, task = "entity_pair_classification", return_tensors = "pt")
#torch.save(inputs, 'inputs_eval.pt')
#test_dataset = MyDataset(inputs, gold_labels_ids)

logging.info("Beginning of evaluation batching")
output_dir = "evalClassic_17Out"
if not os.path.exists(output_dir):
    os.makedirs(output_dir)
output_filename = os.path.join(output_dir, 'results.json')
output_file = open(output_filename, 'w')

batch_size = 10
rel2word = {
            "Na": "Na",
            "P6": "head of government",
            "P17": "country",
            "P19": "place of birth",
            "P20": "place of death",
            "P22": "father",
            "P25": "mother",
            "P26": "spouse",
            "P27": "country of citizenship",
            "P30": "continent",
            "P31": "instance of",
            "P35": "head of state",
            "P36": "capital",
            "P37": "official language",
            "P39": "position held",
            "P40": "child",
            "P50": "author",
            "P54": "member of sports team",
            "P57": "director",
            "P58": "screenwriter",
            "P69": "educated at",
            "P86": "composer",
            "P102": "member of political party",
            "P108": "employer",
            "P112": "founded by",
            "P118": "league",
            "P123": "publisher",
            "P127": "owned by",
            "P131": "located in the administrative territorial entity",
            "P136": "genre",
            "P137": "operator",
            "P140": "religion",
            "P150": "contains administrative territorial entity",
            "P155": "follows",
            "P156": "followed by",
            "P159": "headquarters location",
            "P161": "cast member",
            "P162": "producer",
            "P166": "award received",
            "P170": "creator",
            "P171": "parent taxon",
            "P172": "ethnic group",
            "P175": "performer",
            "P176": "manufacturer",
            "P178": "developer",
            "P179": "series",
            "P190": "sister city",
            "P194": "legislative body",
            "P205": "basin country",
            "P206": "located in or next to body of water",
            "P241": "military branch",
            "P264": "record label",
            "P272": "production company",
            "P276": "location",
            "P279": "subclass of",
            "P355": "subsidiary",
            "P361": "part of",
            "P364": "original language of work",
            "P400": "platform",
            "P403": "mouth of the watercourse",
            "P449": "original network",
            "P463": "member of",
            "P488": "chairperson",
            "P495": "country of origin",
            "P527": "has part",
            "P551": "residence",
            "P569": "date of birth",
            "P570": "date of death",
            "P571": "inception",
            "P576": "dissolved, abolished or demolished",
            "P577": "publication date",
            "P580": "start time",
            "P582": "end time",
            "P585": "point in time",
            "P607": "conflict",
            "P674": "characters",
            "P676": "lyrics by",
            "P706": "located on terrain feature",
            "P710": "participant",
            "P737": "influenced by",
            "P740": "location of formation",
            "P749": "parent organization",
            "P800": "notable work",
            "P807": "separated from",
            "P840": "narrative location",
            "P937": "work location",
            "P1001": "applies to jurisdiction",
            "P1056": "product or material produced",
            "P1198": "unemployment rate",
            "P1336": "territory claimed by",
            "P1344": "participant of",
            "P1365": "replaces",
            "P1366": "replaced by",
            "P1376": "capital of",
            "P1412": "languages spoken, written or signed",
            "P1441": "present in work",
            "P3373": "sibling"}

num_predicted = 0
num_gold = 0
num_correct = 0
this_pair = []
all_pairs = []
list_of_dicts = []
torch.cuda.empty_cache()

logging.info("Evaluation will start now!:")
model.eval()
model.to(device)
for batch_start_idx in trange(0, len(test_examples), batch_size):# len(test_examples) 100
    batch_examples = test_examples[batch_start_idx:batch_start_idx + batch_size]
    texts = [example["text"] for example in batch_examples]
    entity_spans = [example["entity_spans"] for example in batch_examples]
    idxs_entity_pair = [example["idxs_entity_pair"] for example in batch_examples]
    titles = [example["title"] for example in batch_examples]
    #gold_labels = [example["labels"] for example in batch_examples]

    #gold_labels_ids = [c2l.str2int(label) for label in gold_labels]

    for i in range(len(entity_spans)):
        entity_spans[i] = list(entity_spans[i])

    inputs = tokenizer(text=texts, entity_spans=entity_spans, truncation=True, padding = "max_length", max_length = 512, task = "entity_pair_classification", return_tensors = "pt").to(device)
    with torch.no_grad():
        outputs = model(**inputs)
    predicted_indices = outputs.logits.argmax(-1)
    predicted_labels = [c2l.int2str(pred) for pred in predicted_indices.tolist()]
    predicted_relation = [rel2word.get(rel) for rel in predicted_labels]

    for i in range(len(predicted_relation)):
        list_of_dicts.append(dict(
            title=titles[i],
            h_idx=idxs_entity_pair[i][0],
            t_idx = idxs_entity_pair[i][1],
            r = predicted_relation[i]
        ))
    torch.cuda.empty_cache()

json_object = json.dumps(list_of_dicts, indent = 4)
with open("results_classic.json", "w") as outfile:
    outfile.write(json_object)


# 클레어와 물약
# r1 x
# https://www.acmicpc.net/problem/20119

# https://welog.tistory.com/256

import sys
from collections import deque
input = sys.stdin.readline

n, m = map(int, input().split())
graph = [set() for _ in range(n + 1)]
recipe_dict = {}

for _ in range(m):
    data = list(map(int, input().split()))

    if data[-1] not in recipe_dict:
        recipe_dict[data[-1]] = [[data[1:-1], data[0]]]
    else:
        recipe_dict[data[-1]].append([data[1:-1], data[0]])

    for i in range(1, len(data) - 1):
        graph[data[i]].add(data[-1])

l = int(input())
l_list = list(map(int, input().split()))
check = [False] * (n + 1)
result = set()

for i in l_list:
    check[i] = True
    result.add(i)

q = deque(l_list)

while q:
    now = q.popleft()

    for i in graph[now]:
        if check[i]:
            continue

        for idx in range(len(recipe_dict[i])):
            recipe, cnt = recipe_dict[i][idx]

            if now in recipe:
                recipe.remove(now)
                cnt -= 1
                recipe_dict[i][idx] = [recipe, cnt]

                if cnt == 0:
                    check[i] = True
                    q.append(i)
                    result.add(i)

print(len(result))
result = list(result)
result.sort()
print(*result)


import pandas as pd
import numpy as np
import plotly.graph_objects as go
from plotly.subplots import make_subplots
import plotly.io as pio
from matplotlib import cm

# set defaults for charts
pio.templates.default = "plotly_white"

@np.vectorize
def calculate_tax(income):
    brackets = [9950, 40525, 86375, 164925, 209425, 523600]
    rates = [0.10, 0.12, 0.22, 0.24, 0.32, 0.35, 0.37]
    tax = 0
    for i in range(len(brackets)):
        if income > brackets[i]:
            if i == 0:
                tax += rates[i] * brackets[i]
            else:
                tax += rates[i] * (brackets[i] - brackets[i - 1])
        else:
            if i == 0:
                tax += rates[i] * income
            else:
                tax += rates[i] * (income - brackets[i - 1])
            break
    if income > brackets[-1]:
        tax += rates[-1] * (income - brackets[-1])

    return tax

# parameters
variables = ['robert_income', 'isabel_income', 'expenses', 'assets']

# create initial setup DataFrame
data = pd.DataFrame({
    'year': [2023],
    'robert_income': [100000],
    'isabel_income': [200000],
    'expenses': [50000],
    'assets': [800000]
}).set_index('year')

growth_assumptions = {
    'robert_income': 0.0,
    'isabel_income': 0.0,
    'expenses': 0.01,
    'assets': 0.04
}

shocks = {
    2027: {
        'robert_income': (-10000, 'Robert leaves Google'),
        'isabel_income': (-100000, 'Isabel book deals are smaller')
    },
    2030: {
        'expenses': (30000, 'Childcare')
    }
}

volatility = 0.08  # standard deviation of asset growth
simulations = 1000  # number of simulations

# create a DataFrame to hold the future projections
projection = pd.DataFrame(index=range(2023, 2083))

# initialize a DataFrame with simulations for assets
asset_simulations = pd.DataFrame(1 + volatility * np.random.standard_normal(size=(60,10000)),
                                 index=projection.index,
                                 columns=['simulation_'+str(i) for i in range(10000)]
)

# chain all
asset_simulations = asset_simulations.cumprod()

# loop over years
for year in projection.index:
    if year == 2023:
        # handle base year
        for var in variables:
            projection.loc[year, var] = data.loc[2023, var]
            asset_simulations.loc[year] = data.loc[2023, 'assets']
    else:
        # apply growth assumptions and shocks
        for var in variables:
            projection.loc[year, var] = projection.loc[year - 1, var] * (1 + growth_assumptions[var])
            if year in shocks and var in shocks[year]:
                shock, _ = shocks[year][var]
                projection.loc[year, var] += shock

        # calculate household income and savings
        projection.loc[year, 'household_income'] = projection.loc[year, 'robert_income'] + projection.loc[year, 'isabel_income']
        projection.loc[year, 'taxes'] = calculate_tax(projection.loc[year, 'household_income'])
        projection.loc[year, 'net_household_income'] = projection.loc[year, 'household_income'] - projection.loc[year, 'taxes']

        # calculate savings
        projection.loc[year, 'savings'] = projection.loc[year, 'net_household_income'] - projection.loc[year, 'expenses']

        # add savings to assets
        projection.loc[year, 'assets'] += projection.loc[year, 'savings']

        # add volatility to assets
        asset_simulations.loc[year] = projection.loc[year - 1, 'assets'] * (asset_simulations.loc[year])

# plot income, expenses, and savings
fig = go.Figure(layout=go.Layout(template='plotly_white'))

for var in ['robert_income', 'isabel_income', 'expenses', 'savings', 'household_income','net_household_income','taxes']:
    fig.add_trace(go.Scatter(x=projection.index, y=projection[var], mode='lines', name=var))

fig.show()

# plot asset simulations as a fan chart
fig = go.Figure()

percentiles = [1, 5, 20, 50, 80, 95, 99]
colors = [cm.Blues(x) for x in np.linspace(0.01, 1, 7)]

for i in range(len(percentiles)):
    percentile = percentiles[i]
    color = colors[i]
    asset_percentile = asset_simulations.apply(lambda x: np.percentile(x, percentile), axis=1)
    fig.add_trace(go.Scatter(x=asset_percentile.index, y=asset_percentile, fill='tonexty', fillcolor='rgba'+str(color), line_color='rgba'+str(color), name=str(percentile)+'th percentile'))

fig.show()

# plot shocks
all_shock_values = []

for shock_type in ['assets', 'robert_income', 'isabel_income', 'expenses']:
    for year, shocks_in_year in shocks.items():
        if shock_type in shocks_in_year:
            all_shock_values.append(shocks_in_year[shock_type][0])

fig = make_subplots(rows=4, cols=1, shared_xaxes=True, shared_yaxes='rows')

for shock_type, subplot in zip(['assets', 'robert_income', 'isabel_income', 'expenses'], [1, 2, 3, 4]):
    shock_years = []
    shock_values = []
    hover_texts = []  # New list to store hover text labels
    for year, shocks_in_year in shocks.items():
        if shock_type in shocks_in_year:
            shock_years.append(year)
            shock_values.append(shocks_in_year[shock_type][0])
            hover_texts.append(shocks_in_year[shock_type][1])  # Add the hover text label to the list
    fig.add_trace(go.Bar(x=shock_years, y=shock_values, name=shock_type + ' shocks', text=hover_texts, textposition='outside', hovertemplate='%{text}', textfont=dict(color='rgba(0,0,0,0)')), row=subplot, col=1)

fig.update_xaxes(range=[2023, 2082])
fig.update_yaxes(range=[min(all_shock_values), max(all_shock_values)])
fig.update_layout(template='plotly_white')
fig.show()


from os.path import isfile
import json
from db import normalize
from itertools import product

class Settings:

    def __init__(self, user_id):
        self.user_id = user_id
        self.search = {}
        self.match = []

    @staticmethod
    def from_dict(settings: dict) -> tuple:
        return (
            settings.get('search'),
            settings.get('match'),
        )

    @staticmethod
    def to_dict(search, match) -> dict:
        return {
            'search': search,
            'match': match,
        }

    def load_from_file(self) -> bool:
        file_name = f'settings_{self.user_id}.json'
        if not isfile(file_name):
            return False
        with open(file_name, mode='r', encoding='utf-8') as file:
            self.search, self.match = self.from_dict(json.load(file))
            return True

    def save_to_file(self):
        file_name = f'settings_{self.user_id}.json'
        with open(file_name, mode='w', encoding='utf-8') as file:
            json.dump(
                self.to_dict(self.search, self.match),
                file, ensure_ascii=False, indent=4)

    def load_from_vk(self, vk_user: dict) -> bool:
        vk_user['sex'] = {1: 2, 2: 1}.get(vk_user['sex'], 0)

        search_fields = {
            'city', 'country', 'hometown',
            'sex', 'has_photo', 'religion'
        }
        search_fields_with_fix = {
            'universities': 'university',
            'schools': 'school',
            'career': 'company'
        }
        match = {
            'universities', 'schools', 'status', 'activities',
            'interests', 'music', 'movies', 'tv', 'books',
            'games', 'about', 'quotes', 'career', 'military', 'langs',
            'verified', 'sex', 'city', 'country', 'home_town', 'has_photo',
            'has_mobile', 'common_count', 'occupation', 'relation',
            'can_post', 'can_see_all_posts', 'can_see_audio',
            'can_write_private_message', 'can_send_friend_request',
            'is_hidden_from_feed', 'blacklisted', 'blacklisted_by_me',
            'political', 'religion', 'inspired_by', 'people_main',
            'life_main', 'smoking', 'alcohol'
        }

        search_params = {
            field: value for field, value in vk_user.items()
            if field in search_fields
            }

        for field, alias in search_fields_with_fix.items():
            if field in vk_user and len(vk_user[field]) == 1:
                search_params[alias] = vk_user[field][0]

        self.search = search_params.copy()

        self.match = [
            (field, value) for field, value in vk_user.items()
            if field in match and value
        ]

    def load_settings(self, vk):
        if self.load_from_file():
            return
        user = vk.get_user(self.user_id)
        normalize.normalize(user)
        self.load_from_vk(user)

    def make_flat_searc_params(self, searc_params=None):
        if searc_params is None:
            searc_params = self.search
        arrays = [
            product([key], value) for key, value in searc_params.items()
            if isinstance(value, (list, tuple))
            ]
        result = []
        for iteam in map(dict, product(*arrays)):
            new_iteam = searc_params.copy()
            new_iteam.update(iteam)
            result.append(new_iteam)
        return result

    def add_settings(self):
        new_settings = {
            'sort': [0, 1],
            'online': [0, 1]
        }
        self.search.update(new_settings)

    def get_base_searc(self):
        return {
            'sex': self.search.get('sex', [0, 1, 2]),
            'age_from': self.search.get('age_from', 18),
            'age_to': self.search.get('age_to', 25),
            'country': self.search.get('country', 1),
        }


import json
import boto3
from botocore.exceptions import ClientError
import os

region = os.environ['AWS_REGION']
sess = boto3.session.Session(region_name=region)

# def get_bucket_name():
#     ssmClient = sess.client('ssm')
#     response = ssmClient.get_parameter(
#             Name = 'ProserveProject_S3BucketName',
#             WithDecryption = True)
#     return response['Parameter']['Value']

def lambda_handler(event, context):

    s3Client = sess.client('s3')

    # try:
    #     bucketName = get_bucket_name()
    # except ClientError as e:
    #     print(e)
    #     return {
    #         'statusCode': 500,
    #         'body': json.dumps("An error occurred")
    #     }
    bucketName = os.environ['BUCKET_NAME']

    objectKey = json.loads(event['body'])["objectKey"].strip()

    response = s3Client.delete_object(
        Bucket = bucketName,
        Key = objectKey,
        VersionId = "null",
    )

    return {
        'statusCode': 204,
        'headers': {
            'Access-Control-Allow-Headers': '*',
            'Access-Control-Allow-Origin': '*',
            'Access-Control-Allow-Methods': 'OPTIONS,POST,GET,DELETE'
        },
    }


import numpy as np
import cv2
from scipy import ndimage, interpolate

def track(I, J, input_points, total_points, window=(21, 21), min_disp=0.01):

    output = []
    I_gray = cv2.cvtColor(I, cv2.COLOR_BGR2GRAY)
    J_gray = cv2.cvtColor(J, cv2.COLOR_BGR2GRAY)

    #normalization
    I_norm = I_gray/I_gray.max()
    J_norm = J_gray/J_gray.max()

    for points in input_points:
        print('inside calculate')
        d = calculate_new_point(I_norm, J_norm, points[0], points[1], window)
        if d is not None:
            print('output '+str(output))
            output.append((points[0] + d[0], points[1] + d[1]))
    output = np.asarray(output).T
    output = output.astype(int)
    frame = J.copy()
    for point in zip(*total_points[::-1]):
        print('printing new points')
        print(point)
        print(type(point))
        J = cv2.circle(J, point, 3, (0, 0, 255), 10)

    for point in zip(*output[::-1]):
        print('printing new points')
        print(point)
        print(type(point))
        J = cv2.circle(J, point, 3, (0, 0, 255), 10)
    # for point in zip(*output[::-1]):
    #     frame = cv2.circle(frame, point, 3, (255, 0, 0), 4)

    return J, output

def calculate_new_point(I, J, x, y, window):
    displ_tot = np.array([0., 0.]).T

    # The window to evaluate
    win_x = np.arange(x, x + window[0], dtype=float)
    win_y = np.arange(y, y + window[1], dtype=float)

    roi = I[x:x + window[0], y: y + window[1]]

    # Find image gradient in I
    Ix = cv2.Sobel(roi,cv2.CV_64F,1,0,ksize=3)
    Iy = cv2.Sobel(roi,cv2.CV_64F,0,1,ksize=3)

    # Calculate the Hessian matrix
    Ix = Ix.flatten()
    Iy = Iy.flatten()
    A = np.array([Ix, Iy])
    T = A.dot(A.T)
    #T = np.matmul(A, A.T)
    # Check that H is not singular
    if np.linalg.det(T) == 0:
        return None
    T_inv = np.linalg.inv(T)

    # Bilinear interpolation
    x_arr = np.arange(0, J.shape[1])
    y_arr = np.arange(0, J.shape[0])
    J_bilinear = interpolate.interp2d(x_arr, y_arr, J, kind='linear')

    for x in range(35):
        try:
            # Calculate e matrix
            J_window = J_bilinear(win_x + displ_tot[0], win_y + displ_tot[1])
            D = (I[x:x + window[0], y: y + window[1]]-J_window).flatten()
            e = -1*(np.dot(A,D))
            d_temp = np.dot(T_inv, e)
            displ_tot = displ_tot + d_temp

            return displ_tot
        except:
            return None

    # calculate displacement

def compute_corners(img, threshold=0.5):

    img_cpy = img.copy()

    # Grayscale
    img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

    #Ix = Convolution.convolution(img_gray, 'sobel_x')
    #Iy = Convolution.convolution(img_gray, 'sobel_y')
    Ix = cv2.Sobel(img_gray,cv2.CV_64F,1,0,ksize=3)
    Iy = cv2.Sobel(img_gray,cv2.CV_64F,0,1,ksize=3)

    Ix2 = np.square(Ix)
    Iy2 = np.square(Iy)
    Ixdy = Ix*Iy

    #g_Ix2 = Convolution.convolution(dx2, 'gaussian')
    #g_Iy2 = Convolution.convolution(dy2, 'gaussian')
    #g_IxIy = Convolution.convolution(dxdy, 'gaussian')

    g_Ix2 = cv2.GaussianBlur(Ix2, (3,3),0)
    g_Iy2 = cv2.GaussianBlur(Iy2, (3,3),0)
    g_IxIy = cv2.GaussianBlur(Ixdy, (3,3),0)

    R = g_Ix2*g_Iy2 - np.square(g_IxIy) - 0.22*np.square(g_Ix2 + g_Iy2)

    # find all points above threshold
    img_cpy[R>threshold]=[255,0,0]

    return img_cpy, np.where(R > threshold*R.max())

cap = cv2.VideoCapture('Ass_img/MarinaBayatNightDrone.mp4')

# Check if camera opened successfully
if (cap.isOpened()== False):
  print("Error opening video stream or file")

# Capture frame-by-frame
ret, frame = cap.read()
cv2.namedWindow('Frame',cv2.WINDOW_NORMAL)
cv2.resizeWindow('Frame', 900,600)

old_frame, points = compute_corners(frame)

points = np.asarray(points)
total_points = points
print(len(points.T))
cv2.imshow('Frame',old_frame)

# Read until video is completed
while(cap.isOpened()):
    ret, new_frame = cap.read()
    old_frame, points = track(old_frame, new_frame, points.T, total_points)
    cv2.imshow('Frame',old_frame)
    print('points and total points')
    print(points)
    print('total points')
    print(total_points)
    total_points = np.hstack((total_points, points))

    # Press Q on keyboard to  exit
    if cv2.waitKey(25) & 0xFF == ord('q'):
      break

# When everything done, release the video capture object
cap.release()

# Closes all the frames
cv2.destroyAllWindows()
#
# corner_det = Corner_Detector()
# corners = corner_det.compute_corners()