import torch
from tqdm.auto import tqdm
from sklearn.metrics import classification_report, f1_score
from sklearn.metrics import precision_recall_fscore_support as score

class Trainer:
    """
        Class to train and evaluate the model.
    """
    def __init__(self, model, preds_loss_fn, roles_loss_fn, optimizer, device, early_stopping):
        """
            Args:
                - model: the model to train
                - preds_loss_fn: loss for the predicate disambiguation task
                - roles_loss_fn: loss for the role classification task
                - optimizer
                - device
                - early_stopping
        """
        self.model = model
        self.preds_loss_fn = preds_loss_fn
        self.roles_loss_fn = roles_loss_fn
        self.optimizer = optimizer
        self.device = device
        self.model.to(self.device)
        self.early_stopping = early_stopping

    def train(self, train_dataset, valid_dataset, predicate_null_token, role_null_token, epochs=1):
        """
           Train the model.
           Args:
                - train_dataset: dataset to use for the train phase
                - valid_dataset: dataset to use for the validation phase
                - predicate_null_token: the representation of the null token in the vocabulary of predicates
                - epochs: number of epochs to train the model
            Return:
                - train_losses: the loss at each epoch computed on the train dataset
                - valid_losses: the loss at each epoch computed on the validation dataset
                - f1_scores: the f1-score at each epoch computed on the validation dataset
        """
        print('Training ...')
        train_losses = []
        valid_losses = []
        f1_scores = []
        for epoch in range(epochs):
            avg_loss, updates = 0, 0
            self.model.train()
            train_iterator = tqdm(enumerate(train_dataset), desc=f'Epoch {epoch + 1}', total=len(train_dataset))
            for step, sample in train_iterator:
                inputs = sample['sentence'].to(self.device)
                lens = sample['lens'].to(self.device)
                pos_tags = sample['pos_tags'].to(self.device)
                lemmas = sample['lemmas'].to(self.device)
                labels = [label for labels in sample['ground'] for label in labels]
                labels = torch.nn.utils.rnn.pad_sequence(labels, batch_first=True).to(self.device)
                if self.model.task_234:
                    # if the model must do also the predicate disambiguation task, the predicates must be only the position of
                    # them in the sentence
                    labels_predicates = sample['predicates'].to(self.device)
                    predicates = (labels_predicates != predicate_null_token).long()
                    # since labels_predicates is padded with the value of predicate_null_token, a mask built on the lengths of
                    #the sentences is needed to pad the sentences with the token ignored by the losses
                    mask_lens = (torch.arange(labels_predicates.shape[1], dtype=lens.dtype, device=lens.device).expand(len(lens), labels_predicates.shape[1]) < lens.unsqueeze(1)).long()
                    labels_predicates = labels_predicates * mask_lens
                else:
                    predicates = sample['predicates'].to(self.device)

                self.optimizer.zero_grad()

                roles, preds = self.model(inputs, lens, pos_tags, predicates, lemmas, predicate_null_token)
                loss_roles = self.roles_loss_fn(roles.view(-1, self.model.num_roles), labels.view(-1))
                if self.model.task_234:
                    # consider also the loss for the predicate disambiaguation task
                    loss_preds = self.preds_loss_fn(preds.view(-1, self.model.num_predicates), labels_predicates.view(-1))
                # compute a linear combination of the two loss if also the predicate disambiguation task must be done
                loss = 0.4*loss_preds + 0.6*loss_roles if self.model.task_234 else loss_roles

                # compute the loss
                avg_loss = (avg_loss * updates + loss.item()) / (updates + 1)
                updates += 1

                train_iterator.set_description(f'Epoch {epoch + 1} - Train Avg loss: {avg_loss:.4f}')

                loss.backward()
                # use clipping gradient to avoid exploding gradients
                torch.nn.utils.clip_grad_norm_(self.model.parameters(), 2)
                self.optimizer.step()

            # compute the validation loss and the f1-score
            val_loss, f1 = self._validation_loss(valid_dataset, epoch, predicate_null_token, role_null_token)

            # store the value of the metrics to make the plot of their trends
            train_losses.append(avg_loss)
            valid_losses.append(val_loss)
            f1_scores.append(f1)

            # call the early stopping to get the model with the best f1
            self.early_stopping(f1, self.model)
            if self.early_stopping.early_stop:
                print("Early stopping")
                break

        # load the last checkpoint with the best model
        self.model.load_state_dict(torch.load(self.early_stopping.checkpoint_path))
        return train_losses, valid_losses, f1_scores

    def _validation_loss(self, valid_dataset, epoch, predicate_null_token, role_null_token):
        """
            Validate the model.
            Args:
                - valid_dataset: dataset to use for the validation
                - epoch: current epoch
                - predicate_null_token: the representation of the null token in the vocabulary of predicates
            Return:
                - val_avg_loss: value of the loss on the validation dataset at the current epoch
                - f1: value of the f1 on the validation dataset at the current epoch
        """
        # ignore the dropout
        self.model.eval()
        val_avg_loss, val_updates = 0, 0
        all_labels = list()
        all_predictions = list()
        dev_iterator = tqdm(valid_dataset, total=len(valid_dataset))
        with torch.no_grad():
            for sample in dev_iterator:
                inputs = sample['sentence'].to(self.device)
                lens = sample['lens'].to(self.device)
                pos_tags = sample['pos_tags'].to(self.device)
                lemmas = sample['lemmas'].to(self.device)
                labels = [label for labels in sample['ground'] for label in labels]
                labels = torch.nn.utils.rnn.pad_sequence(labels, batch_first=True).to(self.device)
                if self.model.task_234:
                    # if the model must do also the predicate disambiguation task, the predicates must be only the position of them
                    # in the sentence
                    labels_predicates = sample['predicates'].to(self.device)
                    predicates = (labels_predicates != predicate_null_token).long()
                    # since labels_predicates is padded with the value of predicate_null_token, a mask built on the lengths of the
                    #sentences is needed to pad the sentences with the token ignored by the losses
                    mask_lens = (torch.arange(labels_predicates.shape[1], dtype=lens.dtype, device=lens.device).expand(len(lens), labels_predicates.shape[1]) < lens.unsqueeze(1)).long()
                    labels_predicates = labels_predicates * mask_lens
                else:
                    predicates = sample['predicates'].to(self.device)

                roles, preds = self.model(inputs, lens, pos_tags, predicates, lemmas, predicate_null_token)

                loss_roles = self.roles_loss_fn(roles.view(-1, self.model.num_roles), labels.view(-1))
                if self.model.task_234:
                    # consider also the loss for the predicate disambiaguation task
                    loss_preds = self.preds_loss_fn(preds.view(-1, self.model.num_predicates), labels_predicates.view(-1))
                # compute a linear combination of the two loss if also the predicate disambiguation task must be done
                loss = 0.4*loss_preds + 0.6*loss_roles if self.model.task_234 else loss_roles

                # compute the loss on the validation dataset
                val_avg_loss = (val_avg_loss * val_updates + loss.item()) / (val_updates + 1)
                val_updates += 1

                dev_iterator.set_description(f'Epoch {epoch + 1} - Validation Avg loss: {val_avg_loss:.4f}')

                predictions = torch.argmax(roles, dim=-1)
                predictions = predictions.view(-1)
                labels = labels.view(-1)
                # consider only the predictions that are neither pad nor null token
                valid_indices = (labels != 0) & (labels != role_null_token)
                valid_predictions = predictions[valid_indices].cpu()
                valid_labels = labels[valid_indices]

                all_predictions.extend(valid_predictions.tolist())
                all_labels.extend(valid_labels.tolist())
            # compute the f1-score
            f1 = f1_score(all_labels, all_predictions, average='micro', labels=list(set(all_labels)))
        return val_avg_loss, f1

    def evaluate(self, dataset, vocab_roles, vocab_preds, predicate_null_token):
        """
            Evaluate the model using the classification report of both the tasks of predicate disambiguation and role classification
            considering the null token and without the null token.
            Args:
                - dataset: dataset to evaluate the performances of the model
                - vocab_roles: vocabulary of the roles
                - vocab_preds: vocabulary of the predicates
                - predicate_null_token: the representation of the null token in the vocabulary of predicates
        """
        # ignore the dropout
        self.model.eval()
        print('Evaluating ...')
        all_predictions = list()
        all_labels = list()
        all_predictions_preds = list()
        all_labels_preds = list()
        with torch.no_grad():
            for sample in tqdm(dataset, total=len(dataset)):
                inputs = sample['sentence'].to(self.device)
                lens = sample['lens'].to(self.device)
                pos_tags = sample['pos_tags'].to(self.device)
                lemmas = sample['lemmas'].to(self.device)
                labels = [label for labels in sample['ground'] for label in labels]
                labels = torch.nn.utils.rnn.pad_sequence(labels, batch_first=True).to(self.device)
                if self.model.task_234:
                    # if the model must do also the predicate disambiguation task, the predicates must be only the position of them
                    # in the sentence
                    labels_predicates = sample['predicates'].to(self.device)
                    predicates = (labels_predicates != predicate_null_token).long()
                    # since labels_predicates is padded with the value of predicate_null_token, a mask built on the lengths of the
                    # sentences is needed to pad the sentences with the token ignored by the losses
                    mask_lens = (torch.arange(labels_predicates.shape[1], dtype=lens.dtype, device=lens.device).expand(len(lens), labels_predicates.shape[1]) < lens.unsqueeze(1)).long()
                    labels_predicates = labels_predicates * mask_lens
                else:
                    predicates = sample['predicates'].to(self.device)

                roles, preds = self.model(inputs, lens, pos_tags, predicates, lemmas, predicate_null_token)
                # take the maximum value of the representation given by the model and that is the predictions of the roles
                predictions = torch.argmax(roles, dim=-1)
                predictions = predictions.view(-1)
                labels = labels.view(-1)

                if self.model.task_234:
                    # take the maximum value of the representation given by the model and that is the predictions of the predicates
                    predictions_predicates = torch.argmax(preds, dim=-1)
                    predictions_predicates = predictions_predicates.view(-1)
                    labels_predicates = labels_predicates.view(-1)

                # ignore the padding
                valid_indices = labels != 0
                valid_predictions = predictions[valid_indices]
                valid_labels = labels[valid_indices]

                all_predictions.extend(valid_predictions.tolist())
                all_labels.extend(valid_labels.tolist())

                if self.model.task_234:
                    # ignore the padding
                    valid_indices_preds = labels_predicates != 0
                    valid_predictions_pred = predictions_predicates[valid_indices_preds]
                    valid_labels_preds = labels_predicates[valid_indices_preds]

                    all_predictions_preds.extend(valid_predictions_pred.tolist())
                    all_labels_preds.extend(valid_labels_preds.tolist())

        all_labels_idxs = set(all_labels)
        # take the labels predicted ignoring the null token
        labels_idxs = [idx for idx in all_labels_idxs if vocab_roles[idx] != '_']
        labels_names = [vocab_roles[idx] for idx in labels_idxs]
        # take the labels predicted
        labels_idxs_null = [idx for idx in all_labels_idxs]
        labels_names_null = [vocab_roles[idx] for idx in labels_idxs_null]

        if self.model.task_234:
            all_labels_preds_idxs = set(all_labels_preds)
            # take the labels predicted ignoring the null token
            labels_preds_idxs = [idx for idx in all_labels_preds_idxs if vocab_preds[idx] != '_']
            labels_preds_names = [vocab_preds[idx] for idx in labels_preds_idxs]
            # take the labels predicted
            labels_preds_idxs_null = [idx for idx in all_labels_preds_idxs]
            labels_preds_names_null = [vocab_preds[idx] for idx in labels_preds_idxs_null]

        # get the classification reports for the predicate disambiguation task and role classification task both with null token
        # and ignoring the null token
        report = classification_report(all_labels, all_predictions, labels=labels_idxs, target_names=labels_names, digits=3)
        report_null = classification_report(all_labels, all_predictions, labels=labels_idxs_null, target_names=labels_names_null, digits=3)
        preds_report = classification_report(all_labels_preds, all_predictions_preds, labels=labels_preds_idxs, target_names=labels_preds_names, digits=3) if self.model.task_234 else None
        preds_report_null = classification_report(all_labels_preds, all_predictions_preds, labels=labels_preds_idxs_null, target_names=labels_preds_names_null, digits=3) if self.model.task_234 else None
        return preds_report, preds_report_null, report, report_null


import os
from pdb_downloader import PDBDownloader
from primary_sequence import get_seq
from protein_seq import Protein
from msa_converter import msa_convert
# from seqret_runner import SeqretRunner
from mmseqs_runner import MMSeqs2Runner
from dssp_runner import DSSPRunner
from datetime import datetime
from topcons_runner import run_topcons
from consurf_runner import ConsurfRunner

r"""
Protein Spin Label Locator
"""

email = input("Please provide your email address: ")
now = datetime.now()
dt = now.strftime("%m_%d_%Y_%H_%M_%S")

# Ask the user for a PDB ID and then download the PDB file.
pdbD = PDBDownloader()
pdb_id = pdbD.get_user_input()
job_id = f"{dt}_{pdb_id}"
download_path = os.path.join(os.getcwd(), f"{job_id}")
os.mkdir(download_path)
os.chdir(download_path)
PDB_path = pdbD.download_pdb()

# Extract primary sequence from the PDB file
seq = get_seq(PDB_path)

# create AA sequence
protein = Protein(pdb_id=pdb_id)

print("Starting to convert the sequence into FASTA format...")

# # get a fasta file of the sequence
# getF = SeqretRunner(email=email, job_id=job_id, mode="fasta", seq=seq, out_name=f"{pdb_id}_SEQ")
# fasta_path = getF.run_job()

print("Starting to fetch MSA...")
# get MSA file in a3m format and convert it into fasta format
getMSA = MMSeqs2Runner(job=job_id, seq=seq)
getMSA.run_job(pdb_id)
print("MSA fetched. Starting to convert MSA into CLUSTAL format...")
msa_convert(pdb_id)

print("Predicting secondary structures and solvent exposure...")
# Run DSSP
getSecStruct = DSSPRunner(file_name=pdb_id)
getSecStruct.run_job()

protein.check_dssp()
protein.display()

print("Predicting membrane exposure...")
# Run Topcons
run_topcons(pdb_id)

#print("Calculating conservation score...")
# Run Consurf
getCons = ConsurfRunner(pdb_id=pdb_id, email=email, job_id=job_id)
chain_id = getCons.out_chain_id()
getCons.run_job()

# print("Analyzing results...")
# # Read the results fetched by the above tools and modify the Protein object accordingly to record the properties of
# # AminoAcids.
#
# print("Calculating distances between qualified residues...")
# # # Calculate distance
# # runDistance(file_path, protein)
# #
# # output_result(protein)


import os
import secrets
from RESEARCHER import app
from PIL import Image

def save_file(file, path):

	#if image_file:

	# create a random name
	random_hex = secrets.token_hex(20)

	# get file extention via os module
	_, extention = os.path.splitext(file.filename)

	# create image name
	filename = random_hex + extention

	# specify image path
	image_path = os.path.join(app.root_path, path, filename)

	# resize image with pillow and save it
	new_size = (600, 600)
	image = Image.open(image_file)
	image.thumbnail(new_size)
	image.save(image_path)

	## image_file.save(image_path)

	return image_filename

	#return None


from validation import epochVal_metrics_test
from options import args_parser
import os
import sys
import logging
import numpy as np
import copy
from FedAvg import FedAvg
import torch
from torchvision import transforms
from networks.models import DenseNet121
from dataloaders import dataset
from local_supervised import SupervisedLocalUpdate
from local_unsupervised import UnsupervisedLocalUpdate
from torch.utils.data import DataLoader

TRAIN_SIZE = 6500
TEST_SIZE = 1000

def split(dataset, num_users):
    num_items = int(len(dataset) / num_users)
    dict_users, all_idxs = {}, [i for i in range(len(dataset))]
    for i in range(num_users):
        dict_users[i] = set(np.random.choice(all_idxs, num_items, replace=False))
        all_idxs = list(set(all_idxs) - dict_users[i])
    return dict_users

def test(epoch, save_mode_path):
    checkpoint_path = save_mode_path

    checkpoint = torch.load(checkpoint_path)

    net = DenseNet121(out_size=5, mode=args.label_uncertainty, drop_rate=args.drop_rate)
    model = net.cuda()

    model.load_state_dict(checkpoint)

    normalize = transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])

    test_dataset = dataset.CheXpertDataset(
        root_dir=args.root_path,
        csv_file=args.csv_file_train,
        transform=transforms.Compose(
            [
                transforms.Resize((224, 224)),
                transforms.ToTensor(),
                normalize,
            ]
        ),
    )

    test_set_size = TEST_SIZE
    print(f"\nTesting Size: {test_set_size}")
    valid_set_size = len(test_dataset) - test_set_size

    test_dataset, _ = torch.utils.data.random_split(test_dataset, [test_set_size, valid_set_size])

    test_dataloader = DataLoader(
        dataset=test_dataset,
        batch_size=args.batch_size,
        shuffle=False,
        num_workers=8,
        pin_memory=True,
    )

    AUROCs, Accus, F1 = epochVal_metrics_test(model, test_dataloader, thresh=0.4)
    AUROC_avg = np.array(AUROCs).mean()
    Accus_avg = np.array(Accus).mean()
    # Senss_avg = np.array(Senss).mean()
    # Specs_avg = np.array(Specs).mean()
    F1_avg = np.array(F1).mean()

    return AUROC_avg, Accus_avg, F1_avg

snapshot_path = "models/"

AUROCs = []
Accus = []
Senss = []
Specs = []

supervised_user_id = [0, 1]
unsupervised_user_id = [2, 3, 4, 5, 6, 7, 8, 9]
flag_create = False
torch.cuda.empty_cache()

if __name__ == "__main__":

    args = args_parser()

    os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"

    os.environ["CUDA_VISIBLE_DEVICES"] = "7"

    logging.basicConfig(filename="log.txt", level=logging.INFO, format="[%(asctime)s.%(msecs)03d] %(message)s", datefmt="%H:%M:%S")
    logging.info(str(args))
    logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))

    normalize = transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])

    train_dataset = dataset.CheXpertDataset(
        root_dir=args.root_path,
        csv_file=args.csv_file_train,
        transform=dataset.TransformTwice(
            transforms.Compose(
                [
                    transforms.Resize((224, 224)),
                    transforms.RandomAffine(degrees=10, translate=(0.02, 0.02)),
                    transforms.RandomHorizontalFlip(),
                    transforms.ToTensor(),
                    normalize,
                ]
            )
        ),
    )

    # 4000 images[128*128,png] => 35 mins
    train_set_size = TRAIN_SIZE

    logging.info(f"Training Data Size: {train_set_size}")

    valid_set_size = len(train_dataset) - train_set_size

    train_dataset, _ = torch.utils.data.random_split(train_dataset, [train_set_size, valid_set_size])

    dict_users = split(train_dataset, args.num_users)

    net_glob = DenseNet121(out_size=5, mode=args.label_uncertainty, drop_rate=args.drop_rate)

    net_glob.train()

    w_glob = net_glob.state_dict()
    w_locals = []
    trainer_locals = []
    net_locals = []
    optim_locals = []

    for i in supervised_user_id:
        trainer_locals.append(SupervisedLocalUpdate(args, train_dataset, dict_users[i]))

        w_locals.append(copy.deepcopy(w_glob))

        net_locals.append(copy.deepcopy(net_glob).cuda())

        optimizer = torch.optim.Adam(
            net_locals[i].parameters(),
            lr=args.base_lr,
            betas=(0.9, 0.999),
            weight_decay=5e-4,
        )
        optim_locals.append(copy.deepcopy(optimizer.state_dict()))

    for i in unsupervised_user_id:
        trainer_locals.append(UnsupervisedLocalUpdate(args, train_dataset, dict_users[i]))

    # Training Started
    for com_round in range(0, args.rounds):

        logging.info("\nBegin: com_round = {}\n".format(com_round))

        loss_locals = []
        if com_round * args.local_ep < 200:
            for idx in supervised_user_id:
                logging.info(f"Supervised Client: {idx}")
                logging.info("--")

                if com_round * args.local_ep > 20:
                    trainer_locals[idx].base_lr = 3e-4

                local = trainer_locals[idx]

                optimizer = optim_locals[idx]

                w, loss, op = local.train(args, net_locals[idx], optimizer)

                w_locals[idx] = copy.deepcopy(w)

                optim_locals[idx] = copy.deepcopy(op)

                loss_locals.append(copy.deepcopy(loss))

        if com_round * args.local_ep > 20:
            if not flag_create:
                for i in unsupervised_user_id:

                    w_locals.append(copy.deepcopy(w_glob))
                    net_locals.append(copy.deepcopy(net_glob).cuda())

                    optimizer = torch.optim.Adam(
                        net_locals[i].parameters(),
                        lr=args.base_lr,
                        betas=(0.9, 0.999),
                        weight_decay=5e-4,
                    )
                    optim_locals.append(copy.deepcopy(optimizer.state_dict()))
                flag_create = True

            for idx in unsupervised_user_id:
                logging.info(f"Semi-Supervised Client: {idx}")
                logging.info("--")
                local = trainer_locals[idx]
                optimizer = optim_locals[idx]

                w, loss, op = local.train(
                    args,
                    net_locals[idx],
                    optimizer,
                    com_round * args.local_ep,
                    avg_matrix,
                )
                w_locals[idx] = copy.deepcopy(w)
                optim_locals[idx] = copy.deepcopy(op)
                loss_locals.append(copy.deepcopy(loss))

        with torch.no_grad():
            avg_matrix = trainer_locals[0].confuse_matrix
            for idx in supervised_user_id[1:]:
                avg_matrix = avg_matrix + trainer_locals[idx].confuse_matrix
            avg_matrix = avg_matrix / len(supervised_user_id)

        with torch.no_grad():
            w_glob = FedAvg(w_locals)

        net_glob.load_state_dict(w_glob)

        for i in supervised_user_id:
            net_locals[i].load_state_dict(w_glob)

        if com_round * args.local_ep > 20:
            for i in unsupervised_user_id:
                net_locals[i].load_state_dict(w_glob)

        loss_avg = sum(loss_locals) / len(loss_locals)

        logging.info("Loss Avg: {}, Common Round: {}, LR: {} ".format(loss_avg, com_round, args.base_lr))

        save_mode_path = os.path.join(snapshot_path, str(4000), "epoch_" + str(com_round) + ".pth")

        torch.save(net_glob.state_dict(), "models/" + str(4000) + "/epoch_{}.pth".format(str(com_round)))

        AUROC_avg, Accus_avg, F1_avg = test(com_round, save_mode_path)

        logging.info("TEST AUROC: {:6f}, TEST Accus: {:6f}, F1: {:6f}".format(AUROC_avg, Accus_avg, F1_avg))
        print("======================================================")


import logging
import os
import random
# import sys

from flask import Flask, render_template, request, url_for

from fourlang.corenlp_wrapper import CoreNLPWrapper
from fourlang.utils import draw_dep_graph, draw_text_graph, ensure_dir, get_cfg
from fourlang.dependency_processor import Dependencies
from fourlang.dep_to_4lang import DepTo4lang
from fourlang.dict_to_4lang import DictTo4lang
from fourlang.text_to_4lang import TextTo4lang

from pymachine.utils import MachineTraverser

__LOGLEVEL__ = 'INFO'

class FourlangDemo():
    def __init__(self, cfg):
        self.cfg = cfg
        tmp_root = cfg.get('demo', 'tmp_root')
        self.tmp_dir = self.get_tmp_dir_name(tmp_root)
        ensure_dir(self.tmp_dir)
        self.parser_wrapper = CoreNLPWrapper(self.cfg)
        self.dep_to_4lang = DepTo4lang(self.cfg)
        self.dict_to_4lang = DictTo4lang(self.cfg)
        self.dict_to_4lang.read_dict()

    def get_tmp_dir_name(self, tmp_root):
        return tmp_root  # TODO

    def get_dep_table(self, sen_deps):
        t = '
\n'
        for dep in sen_deps:
            t += "
\n"
            for e in (dep[0], dep[1][0], dep[2][0]):
                t += " {0} 	".format(e)
            t += "
\n"
        t += '
\n'
        return t

    def dict_to_4lang_demo(self, word, fn='pic', dep_fn='deps'):
        if word in self.dep_to_4lang.lexicon.lexicon:
            source = '4lang'
        elif word in self.dep_to_4lang.lexicon.ext_lexicon:
            source = 'ext'
        else:
            # OOV
            return None, None, None, None

        machine = self.dep_to_4lang.lexicon.get_machine(word)
        pic_path = draw_text_graph({word: machine}, self.tmp_dir, fn=fn)

        if source == '4lang':
            return source, None, None, os.path.basename(pic_path)
        else:
            entry = demo.dict_to_4lang.dictionary[word]
            definition = entry['senses'][0]['definition']
            deps = map(Dependencies.parse_dependency, definition['deps'])
            dep_path = draw_dep_graph(deps, self.tmp_dir, dep_fn)
            return source, definition['sen'], os.path.basename(dep_path), os.path.basename(pic_path)  # nopep8

    def text_to_4lang_demo(self, text, expand, fn='pic', dep_fn='deps'):
        preproc_sen = TextTo4lang.preprocess_text(text.strip().decode('utf-8'))
        deps, corefs, parse_trees = self.parser_wrapper.parse_text(preproc_sen)
        words2machines = self.dep_to_4lang.get_machines_from_deps_and_corefs(
            deps, corefs)
        # TODO
        orig_machines = set()
        for machine in words2machines.itervalues():
            orig_machines |= set(MachineTraverser.get_nodes(
                machine, names_only=False, keep_upper=True))
        # orig_machines = set([m.printname() for m in words2machines.values()])
        # logging.info(u'orig_machines: {0}'.format(
        #     [m.printname() for m in orig_machines]))
        if expand:
            self.dep_to_4lang.lexicon.expand(words2machines)
        pic_path = draw_text_graph(
            words2machines, self.tmp_dir, fn=fn,
            orig_machines=orig_machines)
        dep_path = draw_dep_graph(deps[0], self.tmp_dir, dep_fn)
        # deps_table = self.get_dep_table(deps[0])
        return os.path.basename(dep_path), os.path.basename(pic_path)

    def backend_test(self):
        u_pic_fn = self.text_to_graph(
            'A man stands in the door', False, 'test_unexpanded')
        logging.info('unexpanded pic drawn to {0}'.format(u_pic_fn))
        e_pic_fn = self.text_to_graph(
            'A man stands in the door', True, 'test_expanded')
        logging.info('expanded pic drawn to {0}'.format(e_pic_fn))

logging.basicConfig(
    level=__LOGLEVEL__,
    format="%(asctime)s : " +
    "%(module)s (%(lineno)s) - %(levelname)s - %(message)s")

cfg_file = os.path.join(os.environ['FOURLANGPATH'], 'conf/demo.cfg')
cfg = get_cfg(cfg_file)
demo = FourlangDemo(cfg)

app = Flask(__name__, static_folder=demo.tmp_dir)
app.debug = True

@app.route('/', methods=['GET'])
def test():
    return render_template('test.html')

@app.route('/dfl', methods=['POST'])
def dfl_demo():
    word = request.form['word']
    source, sen, dep_fn, pic_fn = demo.dict_to_4lang_demo(word)
    if source is None:
        return 'oov'
        # return render_template('oov.html', word=word)
    pic_url = url_for(
        'static', filename=pic_fn, nocache=random.randint(0, 9999))
    if source == '4lang':
        return render_template('dfl_4lang.html', word=word, img_url=pic_url)
    elif source == 'ext':
        dep_url = url_for(
            'static', filename=dep_fn, nocache=random.randint(0, 9999))
        return render_template(
            'dfl_ext.html', word=word, img_url=pic_url, sen=sen,
            dep_url=dep_url)
    else:
        assert False

@app.route('/tfl', methods=['POST'])
def tfl_demo():
    sen = request.form['text']
    dep_fn, pic_fn = demo.text_to_4lang_demo(sen, True)
    pic_url = url_for(
        'static', filename=pic_fn, nocache=random.randint(0, 9999))
    dep_url = url_for(
        'static', filename=dep_fn, nocache=random.randint(0, 9999))
    return render_template(
        'tfl.html', img_url=pic_url, sen=sen, dep_url=dep_url)

if __name__ == "__main__":
    app.debug = True
    app.run(host='0.0.0.0')


import pytorch_lightning as pl
import torch
import torch.nn as nn
import torch.nn.functional as F
import torchmetrics

class Task(pl.LightningModule):
    """Graph Neural Network module"""

    def __init__(
        self,
        model,
        lr,
        force_weight=1,
        shift=0,
        scale=1,
    ):
        super().__init__()
        self.model = model
        self.lr = lr

        self.force_weight = force_weight
        self.shift = shift
        self.scale = scale

        self.energy_train_metric = torchmetrics.MeanAbsoluteError()
        self.energy_valid_metric = torchmetrics.MeanAbsoluteError()
        self.energy_test_metric = torchmetrics.MeanAbsoluteError()
        self.force_train_metric = torchmetrics.MeanAbsoluteError()
        self.force_valid_metric = torchmetrics.MeanAbsoluteError()
        self.force_test_metric = torchmetrics.MeanAbsoluteError()

    def forward(self, graph):
        energy, force = self.pred_energy_and_force(graph)
        return energy, force

    def pred_energy_and_force(self, graph):
        graph.pos = torch.autograd.Variable(graph.pos, requires_grad=True)
        pred_energy = self.model(graph)
        sign = -1.0
        pred_force = (
            sign
            * torch.autograd.grad(
                pred_energy,
                graph.pos,
                grad_outputs=torch.ones_like(pred_energy),
                create_graph=True,
                retain_graph=True,
            )[0]
        )
        return pred_energy.squeeze(-1), pred_force

    def energy_and_force_loss(self, graph, energy, force):
        loss = F.mse_loss(energy, (graph.energy - self.shift) / self.scale)
        loss += self.force_weight * F.mse_loss(force, graph.force / self.scale)
        return loss

    def training_step(self, graph):
        energy, force = self(graph)

        # print("train", energy * self.scale + self.shift - graph.energy)

        loss = self.energy_and_force_loss(graph, energy, force)
        self.energy_train_metric(energy * self.scale + self.shift, graph.energy)
        self.force_train_metric(force * self.scale, graph.force)

        # cur_lr = self.trainer.optimizers[0].param_groups[0]["lr"]
        # self.log("lr", cur_lr, prog_bar=True, on_step=True)
        return loss

    def on_train_epoch_end(self):
        self.log("Energy train MAE", self.energy_train_metric, prog_bar=True)
        self.log("Force train MAE", self.force_train_metric, prog_bar=True)

    @torch.inference_mode(False)
    def validation_step(self, graph, batch_idx):
        energy, force = self(graph)
        # print("valid", energy * self.scale + self.shift - graph.energy)
        self.energy_valid_metric(energy * self.scale + self.shift, graph.energy)
        self.force_valid_metric(force * self.scale, graph.force)

    def on_validation_epoch_end(self):
        self.log("Energy valid MAE", self.energy_valid_metric, prog_bar=True)
        self.log("Force valid MAE", self.force_valid_metric, prog_bar=True)

    @torch.inference_mode(False)
    def test_step(self, graph, batch_idx):
        energy, force = self(graph)
        self.energy_test_metric(energy * self.scale + self.shift, graph.energy)
        self.force_test_metric(force * self.scale, graph.force)

    def on_test_epoch_end(self):
        self.log("Energy test MAE", self.energy_test_metric, prog_bar=True)
        self.log("Force test MAE", self.force_test_metric, prog_bar=True)

    def configure_optimizers(self):
        optimizer = torch.optim.Adam(
            self.parameters(),
            lr=self.lr,
        )

        scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
            optimizer, self.trainer.estimated_stepping_batches
        )

        lr_scheduler_config = {
            "scheduler": scheduler,
            "interval": "step",
            "frequency": 1,
        }
        return [optimizer], [lr_scheduler_config]


# coding:utf-8
import xlrd

if __name__ == '__main__':

    # 打开文件
    workbook = xlrd.open_workbook(r"C:\Users\Administrator\Desktop\1.xls")
    workbook2 = xlrd.open_workbook(r"C:\Users\Administrator\Desktop\2.xls")
        # 获取sheet
    sheet = workbook.sheet_names()[0]

    sheet1 = workbook.sheet_by_index(0)
    sheet2 = workbook2.sheet_by_index(0)
    # print(sheet1.name)
    # # 获取        行和        列数
    # print(sheet1.nrows, sheet1.ncols)

    list1 = sheet1.col_values(0)
    print(sheet1.col_values(0, 1, 2))
    list2 = sheet2.col_values(0)
    print(list1)
    print(list2)
    list3 = []
    for i in range(0,len(list1)-1):
        if list1[i] in list2:
            pass
        else:
            print(list1[i])
            list3.append(list1[i])
    print(len(list3))


import public,os
from json import loads,dumps
class panelApi:
    tokenFile = 'data/token.json'

    #获取Token
    def GetToken(self,get):
        if not os.path.exists(self.tokenFile): return public.returnMsg(False,'错误：当前未开启API接口服务!')
        if os.path.exists(self.tokenFile): self.CreateToken(get);
        token = loads(public.readFile(self.tokenFile))
        return token;

    #设置Token
    def SetToken(self,get):
        if not os.path.exists(self.tokenFile): return public.returnMsg(False,'错误：当前未开启API接口服务!')
        token = loads(public.readFile(self.tokenFile))

        #设置AK/SK
        if hasattr(get,'access_key'):
            token['access_key'] = get.access_key
            token['secret_key'] = get.secret_key

        #设置权限
        if hasattr(get,'rule'):
            token['rule'] = get.rule.split(',')

        #设置IP白名单
        if hasattr(get,'address'):
            token['address'] = get.address.split(',')

        public.writeFile(self.tokenFile,dumps(token))
        public.WriteLog('API','修改API配置成功!');
        return public.returnMsg(True,'设置成功!')

    #初始化API接口
    def CreateToken(self,get):
        token = {}
        token['access_key'] = public.GetRandomString(24)
        token['secret_key'] = public.GetRandomString(48)
        token['rule'] = []
        token['address'] = []
        token['status'] = False
        public.writeFile(self.tokenFile,dumps(token))
        public.WriteLog('API','开启API接口成功!');
        return public.returnMsg(True,'初始化API接口成功!');

    #设置API接口状态
    def SetTokenStatus(self,get):
        if not os.path.exists(self.tokenFile): return public.returnMsg(False,'错误：当前未开启API接口服务!')
        token = loads(public.readFile(self.tokenFile))
        if token['status']:
            token['status'] = False
            public.WriteLog('API','已关闭API服务!');
        else:
            token['status'] = True
            public.WriteLog('API','已开启API服务!');

        public.writeFile(self.tokenFile,dumps(token))
        return public.returnMsg(True,'设置成功!')


from typing import Any

from fastapi import Depends, HTTPException, status
from sqlalchemy.orm import Session

from repository.user import get_by_email

from db.base import get_db
from core.security import JWTBearer, decode_access_token

async def get_current_user(
        db: Session = Depends(get_db),
        token: str = Depends(JWTBearer()),
) -> HTTPException | Any:
    cred_exception = HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Credentials are not valid")
    payload = decode_access_token(token)
    if payload is None:
        raise cred_exception
    email: str = payload.get("sub")
    if email is None:
        raise cred_exception
    user = await get_by_email(db=db, email=email)
    if user is None:
        return cred_exception
    return user


from PyQt5.QtCore import Qt
from PyQt5.QtWidgets import QWidget, QVBoxLayout, QLabel, QHBoxLayout, QPushButton
from PyQt5.QtGui import QPixmap
from tornado.httpclient import HTTPClient
import json
import urllib

class CustomListItem(QWidget):
    def __init__(self, parent=None):
        super(CustomListItem, self).__init__(parent)
        self.textVBox = QVBoxLayout()
        self.remove_button = QPushButton('X', self)
        self.remove_button.setMaximumWidth(30)
        self.titleQLabel = QLabel()
        self.empty_box = QLabel()

        self.hbox_layout = QHBoxLayout()
        self.hbox_layout.addWidget(self.remove_button)
        self.hbox_layout.addWidget(self.empty_box)
        self.hbox_layout.addWidget(self.titleQLabel)
        self.hbox_layout.setSpacing(5)
        self.setLayout(self.hbox_layout)

    def setTitle(self, text):
        self.titleQLabel.setText(text)


# !/usr/bin/env python3
# _*_coding:utf-8_*_
# __author__：FLS

import os
import sys

sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))

import scrapy
import requests
from  urllib.parse import  urlencode
import json
from  A02_360_tupian  import items

class ImageSpider(scrapy.Spider):
    name = 'tupianspider'
    start_urls = ['https://image.so.com/']

    def start_requests(self):
        data={
            "ch":"beauty",
            "listtype":'new',
            "temp":'1',
        }
        base_url='https://image.so.com/zjl?'

        for page in range(1000):
            data['sn']=page*30
            params=urlencode(data)

            url=base_url+params

            yield  scrapy.Request(url,self.parse)

    def parse(self, response):
        result=json.loads(response)
        for image in result.get('list'):
            item=items.imageItem()
            item['id']=image.get('image')


from fastapi.middleware.cors import CORSMiddleware
from fastapi.middleware.gzip import GZipMiddleware
from starlette.middleware.httpsredirect import HTTPSRedirectMiddleware
from starlette.middleware.trustedhost import TrustedHostMiddleware
from starlette.middleware.wsgi import WSGIMiddleware
import time
import logging
from fastapi import Depends, FastAPI, Header, HTTPException, Request
from .routers import items, users
from app.auth.auths import Auth
from fastapi.responses import JSONResponse

def create_app()  ->FastAPI:
    app=FastAPI()

    logger=logging.getLogger('fastapi')

    async def get_token_header(x_token:str=Header(...)):
        if x_token !='fake-super-secret-toke':
            raise HTTPException(status_code=400,detail='X-Token header invalid')

    app.include_router(user.router)
    app.include_router(
        items.router,
        prefix="/items",
        tags=["items"],
        dependencies=[Depends(get_token_header)],
        responses={404: {"description": "Not found"}},
    )

    @app.middleware("http")
    async def process_authorization(request:Request,call_next):
        start_time=time.time()

        if request.url.path == '/login' or request.url.path == '/register':
            logger.info('no jwt verify.')
        else:
            logger.info('jwt verify.')

            result = Auth.identifyAll(request)

            if result['status'] and result['data']:
                user=result['data']['user']
                logger.info('jwt verify success. user: %s ' % user.username)
                request.data.user=user
            else:
                return JSONResponse(content=request)

        response = await call_next(request)

        process_time = time.time() - start_time
        response.headers["X-Process-Time"] = str(process_time)
        return response

    return  app


import numpy as np
import matplotlib.pyplot as plt
import json
import math
import os
from statistics import mean

def levenshtein_distance(obtained, expected):

    size_1 = len(obtained) + 1
    size_2 = len(expected) + 1

    matrix = np.zeros((size_1, size_2))

    for i in range(size_1):
        matrix[i, 0] = i

    for j in range(size_2):
        matrix[0, j] = j

    for i in range(1, size_1):
        for j in range(1, size_2):

            if obtained[i-1] == expected[j-1]:
                matrix[i, j] = min(matrix[i-1, j-1], matrix[i-1, j] + 1, matrix[i, j - 1] + 1)
            else:
                matrix[i, j] = min(matrix[i-1, j] + 1, matrix[i-1, j-1] + 1, matrix[i, j - 1] + 1)

    return matrix[size_1 - 1, size_2 - 1]

def accuracy(obtained, expected):
    distance = levenshtein_distance(obtained, expected)
    max_size = max(len(obtained), len(expected), 1)

    return 1-(distance/max_size)

def main():
    if not os.path.exists("plotResults"):

        # if the demo_folder directory is not present
        # then create it.
        os.makedirs("plotResults")

    # Opening JSON file
    fileObtained = open('obtained.json')
    fileExpected = open('expected.json')

    jsonObtained = json.load(fileObtained)
    jsonExpected = json.load(fileExpected)

    acc_results = dict()
    error_results = dict()
    leagueNumberImages = dict()
    keyNumberImages = dict()

    myKeys = list(jsonObtained.keys())
    myKeys = sorted(myKeys, key=str.casefold)
    jsonObtained = {i: jsonObtained[i] for i in myKeys}

    totalNumberImages = 0

    for league in jsonObtained:

        numberImages = len(jsonObtained[league])

        totalNumberImages += numberImages
        leagueNumberImages[league] = numberImages

        acc_results[league] = dict()
        error_results[league] = dict()

        for image in jsonObtained[league]:
            for key in jsonObtained[league][image]:

                # initialize once
                if key not in acc_results[league]:
                    acc_results[league][key] = dict()
                    error_results[league][key] = dict()

                # control of number of images per key
                if key not in keyNumberImages:
                    keyNumberImages[key] = 1
                else:
                    keyNumberImages[key] += 1

                for method in jsonObtained[league][image][key]:

                    obtained = jsonObtained[league][image][key][method]
                    expected = jsonExpected[league][image][key]

                    acc = 0
                    error = 0

                    acc = accuracy(obtained, expected) * 100
                    if (acc != 100):
                        error = 1

                    if method in acc_results[league][key]:
                        acc_results[league][key][method] += [acc]
                        error_results[league][key][method] += error
                    else:
                        acc_results[league][key][method] = [acc]
                        error_results[league][key][method] = error

    finalResult = dict()
    errorFinalResult = dict()

    for league in acc_results:

        count_1 = count_2 = 0
        index_1 = round(math.sqrt(len(acc_results[league])))
        index_2 = math.ceil(math.sqrt(len(acc_results[league])))

        figure, axis = plt.subplots(index_1, index_2)

        for key in acc_results[league]:
            # initialize once
            if key not in finalResult:
                finalResult[key] = dict()
                errorFinalResult[key] = dict()

            methods = []
            accuracies = []
            errors = []

            ylabel = ""

            points = []

            for method in acc_results[league][key]:
                points.append((method, round(mean(acc_results[league][key][method]), 2), error_results[league][key][method]))

            points.sort(key = lambda x: x[1])

            for point in points:

                method = point[0]
                acc = point[1]
                error = point[2]

                methods.append(method)
                accuracies.append(acc)
                errors.append(error)

                if (not ylabel):
                    ylabel = "[" + str(acc) + "%"
                else :
                    ylabel += ", " + str(acc) + "%"

                if method not in finalResult[key]:
                    finalResult[key][method] = [acc]
                    errorFinalResult[key][method] = error
                else:
                    finalResult[key][method] += [acc]
                    errorFinalResult[key][method] += error

            ylabel += "]"

            axes = None

            if (index_1 == 1):
                axes = axis[count_2]
                plt.subplots_adjust(left=0.1,
                        right=0.93,
                        top=0.8,
                        wspace=0.4,
                        hspace=0.4)
            else:
                axes = axis[count_1, count_2]
                plt.subplots_adjust(left=0.1,
                        right=0.93,
                        wspace=0.3)

            colorBar = 'mediumaquamarine'
            colorText = 'black'
            if (key == 'downDistance'):
                colorText = colorBar = 'maroon'

            bar = axes.bar(methods, accuracies, color=colorBar, width=0.3)
            axes.bar_label(bar, errors, color=colorText)
            axes.set_ylabel(ylabel, color=colorText)
            axes.set_title(key, color=colorText)

            if (count_2 + 1 < index_2):
                count_2 += 1
            else:
                count_2 = 0
                count_1 += 1

        plt.suptitle('Accuracy testing in ' + league + '\n(' + str(leagueNumberImages[league]) + ' image(s))',
                fontsize='large',
                fontweight='bold',
                style='italic',
                family='monospace')
        figure.set_figwidth(10)

        if (index_1 == 1):
            figure.set_figheight(5)
        else:
            figure.set_figheight(9)

        plt.savefig('plotResults/' + league + '.png')
        plt.show()

    # final graph
    count_1 = count_2 = 0
    index_1 = round(math.sqrt(len(finalResult)))
    index_2 = math.ceil(math.sqrt(len(finalResult)))

    figure, axis = plt.subplots(index_1, index_2)

    for key in finalResult:

        methods = []
        accuracies = []
        errors = []

        ylabel = ""

        points = []

        for method in finalResult[key]:
            points.append((method, round(mean(finalResult[key][method]), 2), errorFinalResult[key][method]))

        points.sort(key = lambda x: x[1])

        for point in points:

            method = point[0]
            acc = point[1]
            error = point[2]

            methods.append(method)
            accuracies.append(acc)
            errors.append(error)

            if (not ylabel):
                ylabel = "[" + str(acc) + "%"
            else :
                ylabel += ", " + str(acc) + "%"

        ylabel += "]"

        axes = None

        if (index_1 == 1):
            axes = axis[count_2]
            plt.subplots_adjust(left=0.1,
                    right=0.93,
                    top=0.8,
                    wspace=0.4,
                    hspace=0.4)
        else:
            axes = axis[count_1, count_2]
            plt.subplots_adjust(left=0.1,
                    right=0.93,
                    wspace=0.3)

        colorBar = 'mediumaquamarine'
        colorText = 'black'
        if (key == 'downDistance'):
            colorText = colorBar = 'maroon'

        bar = axes.bar(methods, accuracies, color=colorBar, width=0.3)
        axes.bar_label(bar, errors, color=colorText)
        axes.set_ylabel(ylabel, color=colorText)
        axes.set_title(key + '('+ str(keyNumberImages[key]) + ' image(s))', color=colorText)

        if (count_2 + 1 < index_2):
            count_2 += 1
        else:
            count_2 = 0
            count_1 += 1

    plt.suptitle('Final accuracy testing\n' + '(' + str(totalNumberImages) + ' image(s))',
                fontsize='large',
                fontweight='bold',
                style='italic',
                family='monospace')
    figure.set_figwidth(17)

    if (index_1 == 1):
        figure.set_figheight(5)
    else:
        figure.set_figheight(9)

    plt.savefig('plotResults/final.png')
    plt.show()

if __name__ == "__main__":
    main()


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import io
import os
import zipfile
from tensor2tensor.data_generators import generator_utils
from tensor2tensor.data_generators import image_utils
from tensor2tensor.utils import registry
import tensorflow.compat.v1 as tf

_LSUN_URL = "http://lsun.cs.princeton.edu/htbin/download.cgi?tag=latest&category=%s&set=%s"
_LSUN_DATA_FILENAME = "lsun-%s-%s.zip"

def pil_image():
  import PIL  # pylint: disable=g-import-not-at-top
  return PIL.Image

def _get_lsun(directory, category, split_name):
  """Downloads all lsun files to directory unless they are there."""
  generator_utils.maybe_download(directory,
                                 _LSUN_DATA_FILENAME % (category, split_name),
                                 _LSUN_URL % (category, split_name))

@registry.register_problem
class ImageLsunBedrooms(image_utils.ImageProblem):
  """LSUN Bedrooms."""

  @property
  def num_channels(self):
    """Number of color channels."""
    return 3

  def generate_data(self, data_dir, tmp_dir, task_id=-1):
    """Generates LSUN bedrooms dataset and writes it in data_dir."""
    generator_utils.generate_dataset_and_shuffle(
        self.read_and_convert_to_png(tmp_dir, "train"),
        self.training_filepaths(data_dir, 100, shuffled=False),
        self.read_and_convert_to_png(tmp_dir, "val"),
        self.dev_filepaths(data_dir, 1, shuffled=False))

  def read_and_convert_to_png(self, tmp_dir, split_name):
    """Downloads the datasets, extracts from zip and yields in PNG format."""
    category = "bedroom"
    _get_lsun(tmp_dir, category, split_name)
    filename = _LSUN_DATA_FILENAME % (category, split_name)
    data_path = os.path.join(tmp_dir, filename)
    print("Extracting zip file.")
    zip_ref = zipfile.ZipFile(data_path, "r")
    zip_ref.extractall(tmp_dir)
    zip_ref.close()

    print("Opening database.")
    data_file = os.path.join(tmp_dir,
                             "%s_%s_lmdb/data.mdb" % (category, split_name))

    filename_queue = tf.train.string_input_producer([data_file], num_epochs=1)
    reader = tf.LMDBReader()
    _, webp_image_tensor = reader.read(filename_queue)

    object_count = 0
    with tf.train.MonitoredTrainingSession() as session:
      while True:
        webp_image = session.run(webp_image_tensor)
        object_count += 1
        if object_count % 1000 == 0:
          print("Extracted %d objects." % object_count)
        # Unfortunately Tensorflow doesn't support reading or parsing
        # WebP images, so we have to do it via Image PIL library.
        image = pil_image().open(io.BytesIO(webp_image))
        buf = io.BytesIO()
        width, height = image.size
        image.save(buf, "PNG")
        yield {
            "image/encoded": [buf.getvalue()],
            "image/format": ["png"],
            "image/class/label": [0],
            "image/height": [height],
            "image/width": [width]
        }


"""djangoProject URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/4.1/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path
from utils.views import *
from pixiv_model.views import *

urlpatterns = [
    path("admin/", admin.site.urls),
    #初始化仓库
    path("api/lib/init",LibInit.as_view()),
    #获取仓库列表与新增仓库
    path("api/lib",LibView.as_view()),
    #删除仓库与更新仓库
    path("api/lib/<id>",deleteAndUpdateLibById.as_view()),
    #获取tag
    path("api/tag",getAllTagsWithCount.as_view()),
    #获取图片
    path("api/image",getImages.as_view()),
    #获取图片tag
    path("api/image/<int:pid>",getImageTagsByPid.as_view()),

    path("api/test",test.as_view()),
    #刷新token
    path("api/utils/login/{refresh_token}",refreshToken.as_view()),
    #获取token
    path("/api/utils/login",getRefreshToken.as_view()),
    # path("/api/utils/delete_multi"),
]


import math

import torch

from ptx_classification.data.candid_ptx.candid_ptx import CandidPtxDataModule, CandidPtxDataset
from ptx_classification.utils import get_cache_dir, get_data_dir

data_dir = get_data_dir()
cache_dir = get_cache_dir()

class TestCandidPtxDataModule:
    def test_set_up(self) -> None:
        size_train = 33
        size_val = 11
        size_test = 20
        batch_size = 32

        data_module = CandidPtxDataModule(
            dataset=CandidPtxDataset(root=data_dir, labels_to_use=CandidPtxDataset.LABELS),
            batch_size=batch_size,
            train_val_test_split=(size_train, size_val, size_test),
        )
        train_dl = data_module.train_dataloader()
        val_dl = data_module.val_dataloader()
        val_features, val_labels = next(iter(val_dl))
        for feat in val_features:
            assert torch.min(feat) == 0
            assert torch.max(feat) == 255
            assert val_features[0].size() == (3, 1024, 1024)
            assert val_labels[0].size() == (3,)

        assert val_dl.batch_size == batch_size
        assert len(train_dl) == math.ceil(size_train / batch_size)
        assert len(val_dl) == math.ceil(size_val / batch_size)

n_images = 19236

class TestCandidPtx:
    def test_load_without_cache(self) -> None:
        candid_ptx = CandidPtxDataset(root=data_dir, labels_to_use=CandidPtxDataset.LABELS)
        assert candid_ptx.get_labels().shape == (n_images, 4)

    def test_load_with_cache(self) -> None:
        candid_ptx = CandidPtxDataset(
            root=data_dir, cache_dir=cache_dir, labels_to_use=CandidPtxDataset.LABELS
        )
        assert candid_ptx.get_labels().shape == (n_images, 4)

    def test_load_with_resize(self) -> None:
        candid_ptx = CandidPtxDataset(
            root=data_dir, labels_to_use=CandidPtxDataset.LABELS, resize=(256, 256)
        )
        assert candid_ptx.get_labels().shape == (n_images, 4)
        img = candid_ptx[0][0]
        assert img.size() == (3, 256, 256)

    def test_load_with_only_ct_labels(self) -> None:
        candid_ptx = CandidPtxDataset(root=data_dir, labels_to_use=[CandidPtxDataset.CT])
        assert candid_ptx.get_labels().shape == (n_images, 2)
        assert "patient_id" in candid_ptx.get_labels().columns
        assert CandidPtxDataset.CT in candid_ptx.get_labels().columns


from itertools import cycle
import numpy as np
import torch
import torch.nn as nn
import models
from layers import masked_cross_entropy
from utils import to_var, time_desc_decorator,  pad_and_pack, normal_kl_div, to_bow, bag_of_words_loss, normal_kl_div, embedding_metric
import os
from tqdm import tqdm
from math import isnan
import re
import math
import pickle
#import gensim

word2vec_path = "../datasets/GoogleNews-vectors-negative300.bin"

class Solver(object):
    def __init__(self, config, train_data_loader, eval_data_loader, vocab, is_train=True, model=None):
        self.config = config
        self.epoch_i = 0
        self.train_data_loader = train_data_loader
        self.eval_data_loader = eval_data_loader
        self.vocab = vocab
        self.is_train = is_train
        self.model = model

    @time_desc_decorator('Build Graph')
    def build(self, cuda=True):

        if self.model is None:
            self.model = getattr(models, self.config.model)(self.config)

            # orthogonal initialiation for hidden weights
            # input gate bias for GRUs
            if self.config.mode == 'train' and self.config.checkpoint is None:
                print('Parameter initiailization')
                for name, param in self.model.named_parameters():
                    if 'weight_hh' in name:
                        print('\t' + name)
                        nn.init.orthogonal_(param)

                    # bias_hh is concatenation of reset, input, new gates
                    # only set the input gate bias to 2.0
                    if 'bias_hh' in name:
                        print('\t' + name)
                        dim = int(param.size(0) / 3)
                        param.data[dim:2 * dim].fill_(2.0)

        if torch.cuda.is_available() and cuda:
            self.model.cuda()

        # Overview Parameters
        print('Model Parameters')
        for name, param in self.model.named_parameters():
            print('\t' + name + '\t', list(param.size()))

        if self.config.checkpoint:
            self.load_model(self.config.checkpoint)

        if self.is_train:
            #self.writer = TensorboardWriter(self.config.logdir)
            self.optimizer = self.config.optimizer(
                filter(lambda p: p.requires_grad, self.model.parameters()),
                lr=self.config.learning_rate)

    def save_model(self, epoch):
        """Save parameters to checkpoint"""
        ckpt_path = os.path.join(self.config.save_path, f'{epoch}.pkl')
        print(f'Save parameters to {ckpt_path}')
        torch.save(self.model.state_dict(), ckpt_path)

    def load_model(self, checkpoint):
        """Load parameters from checkpoint"""
        print(f'Load parameters from {checkpoint}')
        epoch = re.match(r"[0-9]*", os.path.basename(checkpoint)).group(0)
        self.epoch_i = int(epoch)
        self.model.load_state_dict(torch.load(checkpoint))

    def write_summary(self, epoch_i):
        epoch_loss = getattr(self, 'epoch_loss', None)
        if epoch_loss is not None:
            self.writer.update_loss(
                loss=epoch_loss,
                step_i=epoch_i + 1,
                name='train_loss')

        epoch_recon_loss = getattr(self, 'epoch_recon_loss', None)
        if epoch_recon_loss is not None:
            self.writer.update_loss(
                loss=epoch_recon_loss,
                step_i=epoch_i + 1,
                name='train_recon_loss')

        epoch_kl_div = getattr(self, 'epoch_kl_div', None)
        if epoch_kl_div is not None:
            self.writer.update_loss(
                loss=epoch_kl_div,
                step_i=epoch_i + 1,
                name='train_kl_div')

        kl_mult = getattr(self, 'kl_mult', None)
        if kl_mult is not None:
            self.writer.update_loss(
                loss=kl_mult,
                step_i=epoch_i + 1,
                name='kl_mult')

        epoch_bow_loss = getattr(self, 'epoch_bow_loss', None)
        if epoch_bow_loss is not None:
            self.writer.update_loss(
                loss=epoch_bow_loss,
                step_i=epoch_i + 1,
                name='bow_loss')

        validation_loss = getattr(self, 'validation_loss', None)
        if validation_loss is not None:
            self.writer.update_loss(
                loss=validation_loss,
                step_i=epoch_i + 1,
                name='validation_loss')

    @time_desc_decorator('Training Start!')
    def train(self):
        epoch_loss_history = []
        for epoch_i in range(self.epoch_i, self.config.n_epoch):
            self.epoch_i = epoch_i
            batch_loss_history = []
            self.model.train()
            n_total_words = 0
            for batch_i, (conversations, conversation_length, sentence_length, images, images_length) in enumerate(tqdm(self.train_data_loader, ncols=80)):
                # conversations: (batch_size) list of conversations
                #   conversation: list of sentences
                #   sentence: list of tokens
                # conversation_length: list of int
                # sentence_length: (batch_size) list of conversation list of sentence_lengths

                input_conversations = [conv[:-1] for conv in conversations]
                target_conversations = [conv[1:] for conv in conversations]

                # flatten input and target conversations
                input_sentences = [sent for conv in input_conversations for sent in conv]
                target_sentences = [sent for conv in target_conversations for sent in conv]
                input_sentence_length = [l for len_list in sentence_length for l in len_list[:-1]]
                target_sentence_length = [l for len_list in sentence_length for l in len_list[1:]]
                input_conversation_length = [l - 1 for l in conversation_length]
                input_images = [i for item in images for i in item]
                input_images_length = [i for item in images_length for i in item]

                input_sentences = to_var(torch.LongTensor(input_sentences))
                target_sentences = to_var(torch.LongTensor(target_sentences))
                input_sentence_length = to_var(torch.LongTensor(input_sentence_length))
                target_sentence_length = to_var(torch.LongTensor(target_sentence_length))
                input_conversation_length = to_var(torch.LongTensor(input_conversation_length))
                input_images = to_var(torch.stack(input_images))
                input_images_length = to_var(torch.LongTensor(input_images_length))

                # reset gradient
                self.optimizer.zero_grad()

                sentence_logits = self.model(
                    input_sentences,
                    input_sentence_length,
                    input_conversation_length,
                    target_sentences,
                    input_images,
                    input_images_length,
                    decode=False)

                batch_loss, n_words = masked_cross_entropy(
                    sentence_logits,
                    target_sentences,
                    target_sentence_length)

                assert not isnan(batch_loss.item())
                batch_loss_history.append(batch_loss.item())
                n_total_words += n_words.item()

                if batch_i % self.config.print_every == 0:
                    tqdm.write(
                        f'Epoch: {epoch_i+1}, iter {batch_i}: loss = {batch_loss.item()/ n_words.item():.3f}')

                # Back-propagation
                batch_loss.backward()

                # Gradient cliping
                torch.nn.utils.clip_grad_norm_(self.model.parameters(), self.config.clip)

                # Run optimizer
                self.optimizer.step()

            epoch_loss = np.sum(batch_loss_history) / n_total_words
            epoch_loss_history.append(epoch_loss)
            self.epoch_loss = epoch_loss

            print_str = f'Epoch {epoch_i+1} loss average: {epoch_loss:.3f}'
            print(print_str)

            if epoch_i % self.config.save_every_epoch == 0:
                self.save_model(epoch_i + 1)

            print('\n<Validation>...')
            self.validation_loss = self.evaluate()

            #if epoch_i % self.config.plot_every_epoch == 0:
            #        self.write_summary(epoch_i)

        self.save_model(self.config.n_epoch)

        return epoch_loss_history

    def generate_sentence(self, input_sentences, input_sentence_length,
                          input_conversation_length, target_sentences, input_images, input_images_length):
        self.model.eval()

        # [batch_size, max_seq_len, vocab_size]
        generated_sentences = self.model(
            input_sentences,
            input_sentence_length,
            input_conversation_length,
            target_sentences,
            input_images,
            input_images_length,
            decode=True)

        # write output to file
        with open(os.path.join(self.config.save_path, 'samples.txt'), 'a') as f:
            f.write(f'<Epoch {self.epoch_i}>\n\n')

            tqdm.write('\n<Samples>')
            for input_sent, target_sent, output_sent in zip(input_sentences, target_sentences, generated_sentences):
                input_sent = self.vocab.decode(input_sent)
                target_sent = self.vocab.decode(target_sent)
                output_sent = '\n'.join([self.vocab.decode(sent) for sent in output_sent])
                s = '\n'.join(['Input sentence: ' + input_sent,
                               'Ground truth: ' + target_sent,
                               'Generated response: ' + output_sent + '\n'])
                f.write(s + '\n')
                print(s)
            print('')

    def evaluate(self):
        self.model.eval()
        batch_loss_history = []
        n_total_words = 0
        for batch_i, (conversations, conversation_length, sentence_length, images, images_length) in enumerate(tqdm(self.eval_data_loader, ncols=80)):
            # conversations: (batch_size) list of conversations
            #   conversation: list of sentences
            #   sentence: list of tokens
            # conversation_length: list of int
            # sentence_length: (batch_size) list of conversation list of sentence_lengths

            input_conversations = [conv[:-1] for conv in conversations]
            target_conversations = [conv[1:] for conv in conversations]

            # flatten input and target conversations
            input_sentences = [sent for conv in input_conversations for sent in conv]
            target_sentences = [sent for conv in target_conversations for sent in conv]
            input_sentence_length = [l for len_list in sentence_length for l in len_list[:-1]]
            target_sentence_length = [l for len_list in sentence_length for l in len_list[1:]]
            input_conversation_length = [l - 1 for l in conversation_length]
            input_images = [i for item in images for i in item]
            input_images_length = [i for item in images_length for i in item]

            with torch.no_grad():
                input_sentences = to_var(torch.LongTensor(input_sentences))
                target_sentences = to_var(torch.LongTensor(target_sentences))
                input_sentence_length = to_var(torch.LongTensor(input_sentence_length))
                target_sentence_length = to_var(torch.LongTensor(target_sentence_length))
                input_conversation_length = to_var(torch.LongTensor(input_conversation_length))
                input_images = to_var(torch.stack(input_images))
                input_images_length = to_var(torch.LongTensor(input_images_length))

            if batch_i == 0:
                self.generate_sentence(input_sentences,
                                       input_sentence_length,
                                       input_conversation_length,
                                       target_sentences,
                                       input_images,
                                       input_images_length)

            sentence_logits = self.model(
                input_sentences,
                input_sentence_length,
                input_conversation_length,
                target_sentences,
                input_images,
                input_images_length
            )

            batch_loss, n_words = masked_cross_entropy(
                sentence_logits,
                target_sentences,
                target_sentence_length)

            assert not isnan(batch_loss.item())
            batch_loss_history.append(batch_loss.item())
            n_total_words += n_words.item()

        epoch_loss = np.sum(batch_loss_history) / n_total_words

        print_str = f'Validation loss: {epoch_loss:.3f}\n'
        print(print_str)

        return epoch_loss

    def test(self):
        self.model.eval()
        batch_loss_history = []
        n_total_words = 0
        for batch_i, (conversations, conversation_length, sentence_length, images, images_length) in enumerate(tqdm(self.eval_data_loader, ncols=80)):
            # conversations: (batch_size) list of conversations
            #   conversation: list of sentences
            #   sentence: list of tokens
            # conversation_length: list of int
            # sentence_length: (batch_size) list of conversation list of sentence_lengths

            input_conversations = [conv[:-1] for conv in conversations]
            target_conversations = [conv[1:] for conv in conversations]

            # flatten input and target conversations
            input_sentences = [sent for conv in input_conversations for sent in conv]
            target_sentences = [sent for conv in target_conversations for sent in conv]
            input_sentence_length = [l for len_list in sentence_length for l in len_list[:-1]]
            target_sentence_length = [l for len_list in sentence_length for l in len_list[1:]]
            input_conversation_length = [l - 1 for l in conversation_length]
            input_images = [i for item in images for i in item]
            input_images_length = [i for item in images_length for i in item]

            with torch.no_grad():
                input_sentences = to_var(torch.LongTensor(input_sentences))
                target_sentences = to_var(torch.LongTensor(target_sentences))
                input_sentence_length = to_var(torch.LongTensor(input_sentence_length))
                target_sentence_length = to_var(torch.LongTensor(target_sentence_length))
                input_conversation_length = to_var(torch.LongTensor(input_conversation_length))
                input_images = to_var(torch.stack(input_images))
                input_images_length = to_var(torch.LongTensor(input_images_length))

            sentence_logits = self.model(
                input_sentences,
                input_sentence_length,
                input_conversation_length,
                target_sentences)

            batch_loss, n_words = masked_cross_entropy(
                sentence_logits,
                target_sentences,
                target_sentence_length,
                input_images,
                input_images_length
            )

            assert not isnan(batch_loss.item())
            batch_loss_history.append(batch_loss.item())
            n_total_words += n_words.item()

        epoch_loss = np.sum(batch_loss_history) / n_total_words

        print(f'Number of words: {n_total_words}')
        print(f'Bits per word: {epoch_loss:.3f}')
        word_perplexity = np.exp(epoch_loss)

        print_str = f'Word perplexity : {word_perplexity:.3f}\n'
        print(print_str)

        return word_perplexity

    def embedding_metric(self):
        word2vec =  getattr(self, 'word2vec', None)
        if word2vec is None:
            print('Loading word2vec model')
            word2vec = gensim.models.KeyedVectors.load_word2vec_format(word2vec_path, binary=True)
            self.word2vec = word2vec
        keys = word2vec.vocab
        self.model.eval()
        n_context = self.config.n_context
        n_sample_step = self.config.n_sample_step
        metric_average_history = []
        metric_extrema_history = []
        metric_greedy_history = []
        context_history = []
        sample_history = []
        n_sent = 0
        n_conv = 0
        for batch_i, (conversations, conversation_length, sentence_length) \
                in enumerate(tqdm(self.eval_data_loader, ncols=80)):
            # conversations: (batch_size) list of conversations
            #   conversation: list of sentences
            #   sentence: list of tokens
            # conversation_length: list of int
            # sentence_length: (batch_size) list of conversation list of sentence_lengths

            conv_indices = [i for i in range(len(conversations)) if len(conversations[i]) >= n_context + n_sample_step]
            context = [c for i in conv_indices for c in [conversations[i][:n_context]]]
            ground_truth = [c for i in conv_indices for c in [conversations[i][n_context:n_context + n_sample_step]]]
            sentence_length = [c for i in conv_indices for c in [sentence_length[i][:n_context]]]

            with torch.no_grad():
                context = to_var(torch.LongTensor(context))
                sentence_length = to_var(torch.LongTensor(sentence_length))

            samples = self.model.generate(context, sentence_length, n_context)

            context = context.data.cpu().numpy().tolist()
            samples = samples.data.cpu().numpy().tolist()
            context_history.append(context)
            sample_history.append(samples)

            samples = [[self.vocab.decode(sent) for sent in c] for c in samples]
            ground_truth = [[self.vocab.decode(sent) for sent in c] for c in ground_truth]

            samples = [sent for c in samples for sent in c]
            ground_truth = [sent for c in ground_truth for sent in c]

            samples = [[word2vec[s] for s in sent.split() if s in keys] for sent in samples]
            ground_truth = [[word2vec[s] for s in sent.split() if s in keys] for sent in ground_truth]

            indices = [i for i, s, g in zip(range(len(samples)), samples, ground_truth) if s != [] and g != []]
            samples = [samples[i] for i in indices]
            ground_truth = [ground_truth[i] for i in indices]
            n = len(samples)
            n_sent += n

            metric_average = embedding_metric(samples, ground_truth, word2vec, 'average')
            metric_extrema = embedding_metric(samples, ground_truth, word2vec, 'extrema')
            metric_greedy = embedding_metric(samples, ground_truth, word2vec, 'greedy')
            metric_average_history.append(metric_average)
            metric_extrema_history.append(metric_extrema)
            metric_greedy_history.append(metric_greedy)

        epoch_average = np.mean(np.concatenate(metric_average_history), axis=0)
        epoch_extrema = np.mean(np.concatenate(metric_extrema_history), axis=0)
        epoch_greedy = np.mean(np.concatenate(metric_greedy_history), axis=0)

        print('n_sentences:', n_sent)
        print_str = f'Metrics - Average: {epoch_average:.3f}, Extrema: {epoch_extrema:.3f}, Greedy: {epoch_greedy:.3f}'
        print(print_str)
        print('\n')

        return epoch_average, epoch_extrema, epoch_greedy

    def generate_for_evaluation(self):
        self.model.eval()
        n_sample_step = self.config.n_sample_step
        n_sent = 0
        fo = open(self.config.pred_path, "w")
        for batch_i, (conversations, conversation_length, sentence_length, images, images_length) in enumerate(tqdm(self.eval_data_loader, ncols=80)):
            # conversations: (batch_size) list of conversations
            #   conversation: list of sentences
            #   sentence: list of tokens
            # conversation_length: list of int
            # sentence_length: (batch_size) list of conversation list of sentence_lengths
            assert len(conversations) == 1
            conversation = conversations[0]
            context = conversation[:-1]
            context_str = ' '.join([self.vocab.decode(sent) for sent in context])
            ground_truth = conversation[-1]
            n_context = len(context)
            sentence_length = sentence_length[0][:-1]
            input_images = [i for item in images for i in item]
            input_images_length = [i for item in images_length for i in item]

            with torch.no_grad():
                context = to_var(torch.LongTensor(context))
                context = context.unsqueeze(0)
                sentence_length = to_var(torch.LongTensor(sentence_length))
                sentence_length = sentence_length.unsqueeze(0)
                input_images = to_var(torch.stack(input_images))
                input_images_length = to_var(torch.LongTensor(input_images_length))

            samples = self.model.generate(context, sentence_length, n_context, input_images, input_images_length)

            samples = samples.data.cpu().numpy().tolist()
            sent = samples[0][0]
            sample = self.vocab.decode(sent)
            ground_truth = self.vocab.decode(ground_truth)

            n_sent += 1

            #print("ground_truth: ", ground_truth)
            #print("gen: ", samples)

            fo.write(context_str + "\t" + ground_truth + "\t" + sample + "\n")
            #print(ground_truth + "\t" + sample)
        print('n_sentences:', n_sent)
        print('\n')
        fo.close()


import copy
import json
import random

from utils import check_results, display_results
import numpy as np
from PIL import Image

def calculate_iou(gt_bbox, pred_bbox):
    """
    calculate iou
    args:
    - gt_bbox [array]: 1x4 single gt bbox
    - pred_bbox [array]: 1x4 single pred bbox
    returns:
    - iou [float]: iou between 2 bboxes
    """
    xmin = np.max([gt_bbox[0], pred_bbox[0]])
    ymin = np.max([gt_bbox[1], pred_bbox[1]])
    xmax = np.min([gt_bbox[2], pred_bbox[2]])
    ymax = np.min([gt_bbox[3], pred_bbox[3]])

    intersection = max(0, xmax - xmin) * max(0, ymax - ymin)
    gt_area = (gt_bbox[2] - gt_bbox[0]) * (gt_bbox[3] - gt_bbox[1])
    pred_area = (pred_bbox[2] - pred_bbox[0]) * (pred_bbox[3] - pred_bbox[1])

    union = gt_area + pred_area - intersection
    return intersection / union, [xmin, ymin, xmax, ymax]

def hflip(img, bboxes):
    """
    horizontal flip of an image and annotations
    args:
    - img [PIL.Image]: original image
    - bboxes [list[list]]: list of bounding boxes
    return:
    - flipped_img [PIL.Image]: horizontally flipped image
    - flipped_bboxes [list[list]]: horizontally flipped bboxes
    """
    # IMPLEMENT THIS FUNCTION
    w,h = img.size

    flipped_img = img.transpose(Image.FLIP_LEFT_RIGHT)
    #flipped_img.show()

    flipped_bboxes = []

    for box in range(len(bboxes)):
        x2 = bboxes[box][2]
        y2 = w - bboxes[box][1]
        x1 = bboxes[box][0]
        y1 = w - bboxes[box][3]

        flipped_bboxes.append([x1, y1, x2, y2])

    return flipped_img, flipped_bboxes

def resize(img, boxes, size):
    """
    horizontal flip of an image and annotations
    args:
    - img [PIL.Image]: original image
    - boxes [list[list]]: list of bounding boxes
    - size [array]: 1x2 array [width, height]
    returns:
    - resized_img [PIL.Image]: resized image
    - resized_boxes [list[list]]: resized bboxes
    """
    # IMPLEMENT THIS FUNCTION
    resized_image = img.resize(size)

    w, h = img.size

    ratio_x = size[0]/h
    ratio_y = size[1]/w

    resized_boxes = np.array(boxes)
    resized_boxes[:,[0, 2]] = (resized_boxes[:,[0, 2]]*ratio_x)
    resized_boxes[:,[1, 3]] = (resized_boxes[:,[1, 3]]*ratio_y)

    return resized_image, resized_boxes

def random_crop(img, boxes, crop_size, min_area=100):
    """
    random cropping of an image and annotations
    args:
    - img [PIL.Image]: original image
    - boxes [list[list]]: list of bounding boxes
    - crop_size [array]: 1x2 array [width, height]
    - min_area [int]: min area of a bbox to be kept in the crop
    returns:
    - cropped_img [PIL.Image]: resized image
    - cropped_boxes [list[list]]: resized bboxes
    """
    # IMPLEMENT THIS FUNCTION
    return cropped_image, cropped_boxes

if __name__ == '__main__':
    # fix seed to check results
    np.random.seed(48)

    # open annotations
    with open('./data/ground_truth.json') as f:
        ground_truth = json.load(f)

    # filter annotations and open image
    filename = 'segment-12208410199966712301_4480_000_4500_000_with_camera_labels_79.png'
    gt_boxes = [g['boxes'] for g in ground_truth if g['filename'] == filename][0]
    gt_classes = [g['classes'] for g in ground_truth if g['filename'] == filename][0]
    img = Image.open(f'./data/images/{filename}')

    # check horizontal flip
    flipped_img, flipped_bboxes = hflip(img, gt_boxes)
    display_results(img, gt_boxes, flipped_img, flipped_bboxes)
    check_results(flipped_img, flipped_bboxes, aug_type='hflip')

    # check resize
    resized_image, resized_boxes = resize(img, gt_boxes, size=[640, 640])
    display_results(img, gt_boxes, resized_image, resized_boxes)
    check_results(resized_image, resized_boxes, aug_type='resize')

    # check random crop
    cropped_image, cropped_boxes, cropped_classes = random_crop(img, gt_boxes, gt_classes, [512, 512], min_area=100)
    display_results(img, gt_boxes, cropped_image, cropped_boxes)
    check_results(cropped_image, cropped_boxes, aug_type='random_crop', classes=cropped_classes)


#!/usr/bin/env python3

import sys
from datetime import datetime
import select
import threading

from OUCHTrade.OUCHSocketHandler import OUCHSocketHandler
from OUCHTrade.OUCHParser import OUCHParser

class OUCHAppServer:

    def __init__(self, host, port):
        self.ouch_server_sock = OUCHSocketHandler()
        self.host = host
        self.port = port
        self.socket_list = []
        self.clients_dict = {}
        self.heartbeat_frequency = 1.0

    def pad_bytes(self, message_tag):
        if message_tag[3] == "alpha":
            message_length = message_tag[2]
            padded_message = message_tag[4].rjust(message_length)
            return bytes(padded_message, encoding='utf-8')
        elif message_tag[3] == "integer":
            message_length = message_tag[2]
            int_value: int
            int_value = message_tag[4]
            padded_message = int_value.to_bytes(message_length, byteorder='big')
            return padded_message

    def create_login_response(self, client_dict):

        login_list = []

        login_list.append(("packet_type", 2, 1, "alpha", "A"))
        login_list.append(("session", 3, 10, "alpha", client_dict["requested_session"]))
        login_list.append(("sequence_number", 13, 20, "alpha", str(client_dict["current_seq_num"]).rjust(20)))

        login_response = b''
        for login_tag in login_list:
            login_response = login_response + self.pad_bytes(login_tag)

        packet_length = len(login_response)
        login_response = packet_length.to_bytes(2, byteorder='big') + login_response

        client_dict["current_seq_num"] += 1

        return login_response

    def create_heartbeat_message(self, client_dict):
        message_list = []

        message_list.append(("packet_type", 2, 1, "alpha", "H"))

        message = b''
        for message_tag in message_list:
            message = message + self.pad_bytes(message_tag)

        packet_length = len(message)
        message = packet_length.to_bytes(2, byteorder='big') + message

        client_dict["current_seq_num"] += 1

        return message

    def create_new_order_ack(self, client_dict, new_order_dict):
        message_list = []

        message_list.append(("packet_type", 2, 1, "alpha", "S"))
        message_list.append(("message_type", 0, 1, "alpha", "A"))
        message_list.append(("timestamp", 1, 8, "integer", self.getSendingTime()))
        message_list.append(("order_token", 9, 4, "integer", new_order_dict["order_token"]))
        message_list.append(("client_reference", 13, 10, "alpha", new_order_dict["client_reference"]))
        message_list.append(("buy_sell_indicator", 23, 1, "alpha", new_order_dict["buy_sell_indicator"]))
        message_list.append(("quantity", 24, 4, "integer", new_order_dict["quantity"]))
        message_list.append(("orderbook_id", 28, 4, "integer", new_order_dict["orderbook_id"]))
        message_list.append(("group", 32, 4, "alpha", new_order_dict["group"]))
        message_list.append(("price", 36, 4, "integer", new_order_dict["price"]))
        message_list.append(("time_in_force", 40, 4, "integer", new_order_dict["time_in_force"]))
        message_list.append(("firm_id", 44, 4, "integer", new_order_dict["firm_id"]))
        message_list.append(("display", 48, 1, "alpha", new_order_dict["display"]))
        message_list.append(("capacity", 49, 1, "alpha", new_order_dict["capacity"]))
        message_list.append(("order_number", 50, 8, "integer", client_dict["current_seq_num"]))
        message_list.append(("minimum_quantity", 58, 4, "integer", new_order_dict["minimum_quantity"]))
        message_list.append(("order_state", 62, 1, "alpha", "L"))
        message_list.append(("order_classification", 63, 1, "alpha", new_order_dict["order_classification"]))
        message_list.append(("cash_margin_type", 61, 1, "alpha", new_order_dict["cash_margin_type"]))

        message = b''
        for message_tag in message_list:
            message = message + self.pad_bytes(message_tag)

        packet_length = len(message)
        message = packet_length.to_bytes(2, byteorder='big') + message

        client_dict["current_seq_num"] += 1

        return message

    def create_replace_ack(self, client_dict, replace_request_dict):
        message_list = []

        message_list.append(("packet_type", 2, 1, "alpha", "S"))
        message_list.append(("message_type", 0, 1, "alpha", "U"))
        message_list.append(("timestamp", 1, 8, "integer", self.getSendingTime()))
        message_list.append(
            ("replacement_order_token", 9, 4, "integer", replace_request_dict["replacement_order_token"]))
        message_list.append(("buy_sell_indicator", 13, 1, "alpha", " "))
        message_list.append(("quantity", 14, 4, "integer", replace_request_dict["quantity"]))
        message_list.append(("orderbook_id", 18, 4, "integer", 0))
        message_list.append(("group", 22, 4, "alpha", " "))
        message_list.append(("price", 26, 4, "integer", replace_request_dict["price"]))
        message_list.append(("time_in_force", 30, 4, "integer", replace_request_dict["time_in_force"]))
        message_list.append(("display", 34, 1, "alpha", replace_request_dict["display"]))
        message_list.append(("order_number", 35, 8, "integer", client_dict["current_seq_num"]))
        message_list.append(("minimum_quantity", 43, 4, "integer", replace_request_dict["minimum_quantity"]))
        message_list.append(("order_state", 47, 1, "alpha", "L"))
        message_list.append(("previous_order_token", 48, 4, "integer", replace_request_dict["existing_order_token"]))

        message = b''
        for message_tag in message_list:
            message = message + self.pad_bytes(message_tag)

        packet_length = len(message)
        message = packet_length.to_bytes(2, byteorder='big') + message

        client_dict["current_seq_num"] += 1

        return message

    def create_cancel_ack(self, client_dict, cancel_request_dict):
        message_list = []

        message_list.append(("packet_type", 2, 1, "alpha", "S"))
        message_list.append(("message_type", 0, 1, "alpha", "C"))
        message_list.append(("timestamp", 1, 8, "integer", self.getSendingTime()))
        message_list.append(("order_token", 9, 4, "integer", cancel_request_dict["order_token"]))
        message_list.append(("decrement_quantity", 13, 4, "integer", cancel_request_dict["quantity"]))
        message_list.append(("order_canceled_reason", 17, 1, "alpha", "U"))

        message = b''
        for message_tag in message_list:
            message = message + self.pad_bytes(message_tag)

        packet_length = len(message)
        message = packet_length.to_bytes(2, byteorder='big') + message

        client_dict["current_seq_num"] += 1

        return message

    def getSendingTime(self):
        now = datetime.now()
        midnight = now.replace(hour=0, minute=0, second=0, microsecond=0)
        nanoseconds = (now - midnight).microseconds * 1000
        return nanoseconds

    def start_sending_heartbeats(self, ouch_client_sock, client_dict):

        heartbeat_thread = threading.Timer(self.heartbeat_frequency, self.start_sending_heartbeats,
                                           [ouch_client_sock, client_dict])
        heartbeat_thread.daemon = True
        heartbeat_thread.start()

        heartbeat = self.create_heartbeat_message(client_dict)
        ouch_client_sock.send(heartbeat)
        print(f"Sending Heartbeat to {ouch_client_sock.sock.getpeername()}: {OUCHParser.parse_ouch_bytes(heartbeat)} ")

    def start(self):

        # Listen for connections from OUCH Client
        self.ouch_server_sock.listen(self.host, self.port)
        self.socket_list.append(self.ouch_server_sock.sock)

        try:
            # Check for incoming messages
            while True:

                read_sockets, _, exception_sockets = select.select(self.socket_list, [], self.socket_list, 0)

                for notified_socket in read_sockets:
                    # If notified socket is a server socket - new connection, accept it
                    if notified_socket == self.ouch_server_sock.sock:
                        # Accept new connection
                        client_socket, client_address = self.ouch_server_sock.sock.accept()
                        print(f"Accepting new connection from {client_address}")
                        self.socket_list.append(client_socket)
                    else:
                        # Receive messages from client
                        ouch_client_sock = OUCHSocketHandler(notified_socket)
                        received_messages = ouch_client_sock.receive()

                        for received_message in received_messages:

                            ouch_dict = OUCHParser.parse_ouch_bytes(received_message)

                            if ouch_dict["packet_type"] == "L":
                                # Found a login request, send a login response
                                print("Received Login Request")
                                requested_session = ouch_dict["requested_session"]

                                self.clients_dict = {
                                                    notified_socket:
                                                    {
                                                        "requested_session": requested_session,
                                                        "current_seq_num": 1
                                                    }
                                }

                                login_response = self.create_login_response(self.clients_dict[notified_socket])
                                ouch_client_sock.send(login_response)
                                print("Sent Login Response")
                                # Start sending Heartbeats
                                self.start_sending_heartbeats(ouch_client_sock, self.clients_dict[notified_socket])
                            elif ouch_dict["packet_type"] == "U":
                                if "message_type" in ouch_dict:
                                    if ouch_dict["message_type"] == "O":
                                        # Found a new order, send a new order ack
                                        print("Received New Order:" + str(ouch_dict))
                                        client_dict = self.clients_dict[notified_socket]
                                        new_order_ack = self.create_new_order_ack(client_dict, ouch_dict)
                                        ouch_client_sock.send(new_order_ack)
                                        print(f"Sending New Order Ack to {ouch_client_sock.sock.getpeername()}: {OUCHParser.parse_ouch_bytes(new_order_ack)} ")
                                    elif ouch_dict["message_type"] == "U":
                                        # Found a replace order, send a replace result
                                        print("Received Replace Order:" + str(ouch_dict))
                                        client_dict = self.clients_dict[notified_socket]
                                        replace_result = self.create_replace_ack(client_dict, ouch_dict)
                                        ouch_client_sock.send(replace_result)
                                        print(f"Sending Order Replaced to {ouch_client_sock.sock.getpeername()}: {OUCHParser.parse_ouch_bytes(replace_result)} ")
                                    elif ouch_dict["message_type"] == "X":
                                        # Found a cancel order, send a cancel result
                                        print("Received Cancel Order:" + str(ouch_dict))
                                        client_dict = self.clients_dict[notified_socket]
                                        cancel_result = self.create_cancel_ack(client_dict, ouch_dict)
                                        ouch_client_sock.send(cancel_result)
                                        print(f"Sending Order Canceled to {ouch_client_sock.sock.getpeername()}: {OUCHParser.parse_ouch_bytes(cancel_result)} ")

        except KeyboardInterrupt:
            print("caught keyboard interrupt, exiting")
        finally:
            for sock in self.socket_list:
                sock.close()

if __name__ == "__main__":

    if len(sys.argv) != 3:
        print("usage:", sys.argv[0], "<host> <port>")
        sys.exit(1)

    main_host, main_port = sys.argv[1], int(sys.argv[2])

    ouch_app_server = OUCHAppServer(main_host, main_port)

    # Start the OUCH Server
    ouch_app_server.start()


import torch
from torchvision.datasets import MNIST
from torchvision import transforms
from torch.utils.data import DataLoader
from torch import optim
import torch.nn as nn

import wandb

from models import Net

def main():
    wandb.init(project="educ_ml")
    # 訓練データを取得
    mnist_train = MNIST("./mydata",
                        train=True, download=True,
                        transform=transforms.ToTensor())
    # テストデータの取得
    mnist_test = MNIST("./mydata",  # 保存先フォルダの指定
                       train=False, download=True,
                       transform=transforms.ToTensor())
    print("訓練データの数:", len(mnist_train), "テストデータの数:", len(mnist_test))

    # DataLoaderの設定
    batch_size = 256
    train_loader = DataLoader(mnist_train,
                              batch_size=batch_size,
                              shuffle=True)
    test_loader = DataLoader(mnist_test,
                             batch_size=len(mnist_test),
                             shuffle=False)

    dev = torch.device("cuda:0" if torch.cuda.is_available()
                       else "cpu")  # GPUが使えたらGPU、使えなかったらCPU
    net = Net()
    net = net.to(dev)  # GPU対応

    # 交差エントロピー誤差関数
    loss_fnc = nn.CrossEntropyLoss()

    lr = 0.01
    epochs = 10
    # SGD
    optimizer = optim.SGD(net.parameters(), lr=lr)
    wandb.config.lr = lr
    # 損失のログ
    record_loss_train = []
    record_loss_test = []

    for i in range(epochs):  # 10エポック学習
        net.train()  # 訓練モードへ
        loss_train = 0
        for j, (x, t) in enumerate(train_loader):  # ミニバッチ(x,t)を取り出す
            x, t = x.to(dev), t.to(dev)  # GPUのメモリに配置する
            y = net(x)
            loss = loss_fnc(y, t)
            loss_train += loss.item()  # ミニバッチなので、誤差を蓄積させていく
            optimizer.zero_grad()  # RNNではためることもあるが普通は初めに勾配を0初期化必須
            loss.backward()  # 逆伝播してパラメタを計算
            optimizer.step()  # 計算した値でパラメタを更新
        loss_train /= j+1  # ループから抜けたらロスの平均を計算
        record_loss_train.append(loss_train)
        print("[Epoch %d/%d] [Batch %d/%d] [loss: %f]"
              % (i, epochs, j+1, len(train_loader), loss.item())
              )
        wandb.log({'train_loss': loss})

    correct = 0
    total = 0
    net.eval()  # 評価モードへ
    for i, (x, t) in enumerate(test_loader):
        x, t = x.to(dev), t.to(dev)   # GPU対応
        x = x.view(-1, 784)
    with torch.no_grad():
        y = net(x)
        test_loss = loss_fnc(y, t)
    correct += (y.argmax(1) == t).sum().item()
    total += len(x)
    print("正解率:", str(correct/total*100) + "%")
    wandb.log({'test_loss': test_loss})
    wandb.save('models.py')

if __name__ == '__main__':
    main()


from rest_framework import serializers

from item.serializers import ItemSerializer
from . import models

class CategorySerializer(serializers.ModelSerializer):
    items = ItemSerializer(many=True, read_only=True)

    class Meta:
        # fields는 데이터베이스 속성을 제어합니다.
        fields =(
            'no',
            'mall_no',
            'name',
            'desc',
            'register_datetime',
            'update_information_datetime',
            'items',
        )
        model = models.Category