####################################################################################
# Estimator Models
# Contains different estimators
# Each Estimator Contains:
#   -A set of required conditional distributions (nuisance parameters) that must be trained
#   -The set of parameters that are shared between nuisance parameter models (embeddings, encoders)
#   -A way to combine the estimator conditional to compute quantity of interest
####################################################################################
import torch
import torch.nn as nn
import numpy as np
import math
import causalchains.models.conditional_models as cmodels
from causalchains.models.encoders.cnn_encoder import CnnEncoder
from causalchains.models.encoders.average_encoder import AverageEncoder
from causalchains.models.encoders.onehot_encoder import OneHotEncoder
from causalchains.models.encoders.rnn_encoder import RnnEncoder
from causalchains.utils.data_utils import PAD_TOK
import logging

EXP_OUTCOME_COMPONENT="_exp_outcome_component_"
PROPENSITY_COMPONENT="_propensity_component_"

class Estimator(nn.Module):
    def __init__(self, event_embed_size, text_embed_size, event_encoder_outsize, text_encoder_outsize, evocab, tvocab, config):
        super(Estimator, self).__init__()

        #Pass in None for event_embed_size to just use one hot encodings for events (make sure event_encoder_outsize is the correct size)
        #Pass in None for event_encoder_outsize if not using event information

        self.event_embed_size = event_embed_size
        self.text_embed_size = text_embed_size
        self.event_encoder_outsize= event_encoder_outsize
        self.text_encoder_outsize= text_encoder_outsize

        evocab_size = len(evocab.stoi.keys())
        tvocab_size = len(tvocab.stoi.keys())
        e_pad = evocab.stoi[PAD_TOK]
        t_pad = tvocab.stoi[PAD_TOK]

        if self.event_embed_size is not None:
            self.event_embeddings = nn.Embedding(evocab_size, self.event_embed_size, padding_idx=e_pad)
        else:
            self.event_embeddings = None

        self.text_embeddings = nn.Embedding(tvocab_size, self.text_embed_size, padding_idx=t_pad)
        if config.use_pretrained:
            logging.info("Estimator: Using Pretrained Word Embeddings")
            self.text_embeddings.weight.data = tvocab.vectors

        if self.event_encoder_outsize is not None:
            if self.event_embed_size is not None and config.rnn_event_encoder:
                logging.info("Estimator: Using RNN Event Encoder")
                self.event_encoder = RnnEncoder(self.event_embed_size, self.event_encoder_outsize)
            elif self.event_embed_size is not None:
                self.event_encoder = AverageEncoder(self.event_embed_size)
            else:
                assert event_encoder_outsize == len(evocab.itos), "event_encoder_outsize incorrectly specified for OneHot, should be vocab size"
                self.event_encoder = OneHotEncoder(len(evocab.itos), pad_idx=e_pad)
        else:
            self.event_encoder = None

        if self.text_encoder_outsize is not None:
            self.text_encoder = CnnEncoder(self.text_embed_size,
                                             num_filters = self.text_embed_size,
                                             output_dim = self.text_encoder_outsize)
        else:
            self.text_encoder = None

    def forward(self, instance):
        """
        Params:
            Should take in a example from data_utils.InstanceDataset which will contain:
                    .e1, Tensor [batch]
                    .e2, Tensor [batch]
                    .e1_text, Tensor [batch, max_size]
                    .e1prev_intext, Tensor[batch, max_size2]
        Outputs:
            (dictonary) Output Dictionary: A dictonary mapping a component name to the logit outputs of each
            component (for example, expected_outcome -> logits outpus of expected outcome)

        """
        raise NotImplementedError

class FineTuneEstimator(nn.Module):  #Assume using rnn encoder for events, cnn encoder for text
    'An estimator with components like the event embeddings, event encoder, text encoder,... pretrained, fixed, and passed in'
    def __init__(self, config, old_model):
        """
        Params:
            old_model (Estimator) : The previous Estimator model we are fine tunning on
        """
        super(FineTuneEstimator, self).__init__()
        self.event_embeddings = old_model.event_embeddings
        self.text_embeddings = old_model.text_embeddings
        self.event_encoder = old_model.event_encoder
        self.text_encoder = old_model.text_encoder
        self.event_embed_size = self.event_embeddings.weight.shape[1]
        self.text_embed_size = self.text_embeddings.weight.shape[1]
        self.event_encoder_outsize= self.event_encoder.output_dim
        self.text_encoder_outsize= self.text_encoder.output_dim

        self.out_event_encoder = AverageEncoder(self.event_embed_size)
        #self.out_event_encoder = OneHotEncoder(self.event_embeddings.weight.shape[0], pad_idx=self.event_embeddings.padding_idx)

class NaiveAdjustmentEstimator(Estimator):
    'Estimate ACE with Backdoor adjustment without considering any previous events'
    def __init__(self, config, evocab, tvocab):
        super(NaiveAdjustmentEstimator, self).__init__(config.event_embed_size,
                                                       config.text_embed_size,
                                                       event_encoder_outsize=None,
                                                       text_encoder_outsize=config.text_enc_output,
                                                       evocab=evocab, tvocab=tvocab, config=config)

        self.expected_outcome = cmodels.ExpectedOutcome(self.event_embeddings,
                                                             self.text_embeddings,
                                                             event_encoder=None,
                                                             text_encoder=self.text_encoder,
                                                             evocab=evocab,tvocab=tvocab,
                                                             config=config)

        assert not self.expected_outcome.includes_e1prev_intext()
        assert not self.expected_outcome.onehot_events()

    def forward(self, instance):
        exp_out = self.expected_outcome(instance)
        return {EXP_OUTCOME_COMPONENT: exp_out}

class AdjustmentEstimator(FineTuneEstimator):
    'Estimate ACE with Backdoor adjustment considering previous events that occured in and out of text, used with finetunning'
    def __init__(self, config, evocab, tvocab, old_model):
        super(AdjustmentEstimator, self).__init__(config, old_model)

        mlp_layer = old_model.expected_outcome.logits_mlp #reuse part of previous last layer
        self.expected_outcome = cmodels.ExpectedOutcome(self.event_embeddings,
                                                             self.text_embeddings,
                                                             event_encoder=self.event_encoder,
                                                             text_encoder=self.text_encoder,
                                                             evocab=evocab,tvocab=tvocab,
                                                             config=config,out_event_encoder=self.out_event_encoder, old_mlp_layer=mlp_layer)

        assert self.expected_outcome.includes_e1prev_intext()

    def forward(self, instance):
        exp_out = self.expected_outcome(instance)
        return {EXP_OUTCOME_COMPONENT: exp_out}

class SemiNaiveAdjustmentEstimator(Estimator):
    'Estimate ACE with Backdoor adjustment considering previous events that occured in text (but not those that didnt appear in text)'
    def __init__(self, config, evocab, tvocab):
        super(SemiNaiveAdjustmentEstimator, self).__init__(config.event_embed_size,
                                                       config.text_embed_size,
                                                       event_encoder_outsize=config.rnn_hidden_dim, #assume using rnn event encoder
                                                       text_encoder_outsize=config.text_enc_output,
                                                       evocab=evocab, tvocab=tvocab, config=config)

        self.expected_outcome = cmodels.ExpectedOutcome(self.event_embeddings,
                                                             self.text_embeddings,
                                                             event_encoder=self.event_encoder,
                                                             text_encoder=self.text_encoder,
                                                             evocab=evocab,tvocab=tvocab,
                                                             config=config)

        assert self.expected_outcome.includes_e1prev_intext()
        assert not self.expected_outcome.onehot_events()

    def forward(self, instance):
        exp_out = self.expected_outcome(instance)
        return {EXP_OUTCOME_COMPONENT: exp_out}

class SemiNaiveAdjustmentEstimatorOneHotEvents(Estimator):
    'Estimate ACE with Backdoor adjustment considering previous events that occured in text (but not those that didnt appear in text)'
    def __init__(self, config, evocab, tvocab):
        super(SemiNaiveAdjustmentEstimatorOneHotEvents, self).__init__(None,
                                                       config.text_embed_size,
                                                       event_encoder_outsize=len(evocab.itos),
                                                       text_encoder_outsize=config.text_enc_output,
                                                       evocab=evocab, tvocab=tvocab, config=config)

        self.expected_outcome = cmodels.ExpectedOutcome(self.event_embeddings,
                                                             self.text_embeddings,
                                                             event_encoder=self.event_encoder,
                                                             text_encoder=self.text_encoder,
                                                             evocab=evocab,tvocab=tvocab,
                                                             config=config)

        assert self.expected_outcome.includes_e1prev_intext()
        assert self.expected_outcome.onehot_events()

    def forward(self, instance):
        exp_out = self.expected_outcome(instance)
        return {EXP_OUTCOME_COMPONENT: exp_out}


from Net import Net
import torch
import torch.nn as nn
import torch.optim as optim
from typing import List
from pathlib import Path
import os

class ModelTrainEval:

    def __init__(self):

        self.net = Net()
        self.criterion = nn.MSELoss()
        self.optimizer = optim.Adam(self.net.parameters(), lr=0.001)

    def train_net(self, num_epochs: int, train_loader: torch.utils.data.DataLoader) -> List[float]:

        self.net.train()

        losses_over_time = []

        for epoch in range(num_epochs):

            running_loss = 0.0

            for batch_i, data in enumerate(train_loader, 0):

                # getting inputs and labels from train_loader
                inputs, labels = data

                # zeroing gradient
                self.optimizer.zero_grad()

                outputs = self.net(inputs.float())

                loss = self.criterion(outputs, labels.float())

                # calculating gradients for the iteration
                loss.backward()

                # taking a step based on calculated gradients
                self.optimizer.step()

                running_loss += loss.item()

                # printing the loss every 2000 mini batches
                if batch_i % 2000 == 1999:
                    print(f'[{epoch + 1}, {batch_i + 1:5d}] training loss: {running_loss/2000:.3f}')
                    losses_over_time.append(running_loss)
                    running_loss = 0.0

        return losses_over_time

        print('Finished Training')

    def eval(self, test_loader: torch.utils.data.DataLoader) -> float:

        self.net.eval()

        test_loss_total = 0

        for data in test_loader:

            test_image, test_label = data

            y_pred = self.net(test_image.float())

            test_loss = self.criterion(y_pred, test_label.float())

            test_loss_total += test_loss.item()

        print('Average test loss is {}'.format(test_loss_total/(len(test_loader) * test_loader.batch_size)))

        return test_loss_total/(len(test_loader) * test_loader.batch_size)

    def save_model(self, model_name: str):

        model_path = Path(os.getcwd()) / "Models" / f"{model_name}.pth".format(model_name)

        torch.save(self.net.state_dict(), model_path)


# Pandigital products

# Problem 32
# We shall say that an n-digit number is pandigital if it makes use of all the digits 1 to n exactly once; for example,
# the 5-digit number, 15234, is 1 through 5 pandigital.
# The product 7254 is unusual, as the identity, 39 × 186 = 7254, containing multiplicand, multiplier, and product is 1 through 9 pandigital.
# Find the sum of all products whose multiplicand/multiplier/product identity can be written as a 1 through 9 pandigital.
# HINT: Some products can be obtained in more than one way so be sure to only include it once in your sum.
# https://projecteuler.net/problem=32

import datetime
import re

def IsPanDigit(i, j, k):
    s = str(i) + str(j) + str(k)
    if len(s) != 9: return False
    m = re.findall(r'(\d).*\1', s)
    if len(m) != 0:
        return False
    a = [int(s1) for s1 in s]
    a.sort()
    res = (a == [1,2,3,4,5,6,7,8,9])
    return res
def IsSingleNotZeroDigit(n):
    s = str(n)
    if s.find("0") != -1: return False
    if len(s) == 1:
        return True
    # trying to find duplicate numbers
    m = re.findall(r'(\d).*\1', s)
    return (len(m) == 0)
def ContainTheSameDigits(i, j):
    si = str(i)
    sj = str(j)
    for s in si:
        if s in sj:
            return True
    return False

start_time = datetime.datetime.now()
a = []
for i in range(10000):
    if IsSingleNotZeroDigit(i):
        a.append(i)
stop_time = datetime.datetime.now()
print(stop_time - start_time)
start_time = datetime.datetime.now()
b = {}
b[7254] = '39,186'
L = len(a)
i1 = 0
dt1 = stop_time - start_time
dt2 = stop_time - start_time
dt3 = stop_time - start_time
dt4 = stop_time - start_time
while i1 < L:
    i2 = i1
    while i2 < L - 1:
        i2 += 1
        m1 = a[i1]
        m2 = a[i2]
        k = m1 * m2
        if len(str(k)) != 4:
            continue
        t1 = datetime.datetime.now()
        if ContainTheSameDigits(m1, m2):
            dt1 += datetime.datetime.now() - t1
            continue
        dt1 += datetime.datetime.now() - t1
        t1 = datetime.datetime.now()
        if not IsSingleNotZeroDigit(k):
            dt2 += datetime.datetime.now() - t1
            continue
        dt2 += datetime.datetime.now() - t1
        t1 = datetime.datetime.now()
        if (k in b):
            dt3 += datetime.datetime.now() - t1
            continue
        dt3 += datetime.datetime.now() - t1
        t1 = datetime.datetime.now()
        if IsPanDigit(m1, m2, k):
            b[k] = str(m1) + ',' + str(m2)
        dt4 += datetime.datetime.now() - t1
    i1 += 1
a = list(b)
print('ContainTheSameDigits - ',dt1)
print('IsSingleNotZeroDigit - ',dt2)
print('In B - ',dt3)
print('IsPanDigit - ',dt4)
stop_time = datetime.datetime.now()
print('Overal - ',stop_time - start_time)
print("{0:,d}".format(sum(a)))
#print("{0:,d}".format(len(result)))
print(b)


import os
import json

import pandas as pd

from aideme.explore import ExplorationManager, PartitionedDataset

from aideme.active_learning import KernelVersionSpace
from aideme.active_learning.dsm import FactorizedDualSpaceModel
from aideme.initial_sampling import random_sampler

import src.routes.points
from src.routes.create_manager import (
    encode_and_normalize,
    compute_indexes_mapping,
    compute_partition_in_encoded_indexes,
)
from src.routes.endpoints import (
    INITIAL_UNLABELED_POINTS,
    NEXT_UNLABELED_POINTS,
)

from tests.routes.data_points import (
    SIMPLE_MARGIN_CONFIGURATION,
    VERSION_SPACE_CONFIGURATION,
    FACTORIZED_SIMPLE_MARGIN_CONFIGURATION,
    FACTORIZED_VERSION_SPACE_CONFIGURATION,
)

TEST_DATASET_PATH = os.path.join(
    __file__.split(sep="tests")[0], "tests", "data", "cars_raw_20.csv"
)
SEPARATOR = ","
SELECTED_COLS = [2, 3]

def test_get_initial_points_to_label(client, monkeypatch):
    def use_config(configuration, column_ids):
        response = client.post(
            INITIAL_UNLABELED_POINTS,
            data={
                "configuration": json.dumps(configuration),
                "columnIds": json.dumps(column_ids),
            },
        )
        points_to_label = json.loads(response.data)

        assert isinstance(points_to_label, list)
        assert len(points_to_label) == 3
        assert isinstance(points_to_label[0], int)

    monkeypatch.setattr(
        src.routes.points,
        "get_dataset_path",
        lambda: TEST_DATASET_PATH,
    )
    monkeypatch.setattr(src.routes.points.cache, "get", lambda key: ",")

    use_config(SIMPLE_MARGIN_CONFIGURATION, column_ids=SELECTED_COLS)
    use_config(VERSION_SPACE_CONFIGURATION, column_ids=SELECTED_COLS)
    use_config(FACTORIZED_SIMPLE_MARGIN_CONFIGURATION, column_ids=[1, 3, 2])
    use_config(FACTORIZED_VERSION_SPACE_CONFIGURATION, column_ids=[1, 3, 2, 3])

def test_get_next_points_to_label(client, monkeypatch):
    cases = [
        {
            "selected_columns": SELECTED_COLS,
            "partition": None,
            "labeled_points": [
                {"id": 3, "label": 1},
                {"id": 9, "label": 1},
                {"id": 11, "label": 0},
            ],
        },
        {
            "selected_columns": [1, 2, 3],
            "partition": [[0, 2], [1, 2]],
            "labeled_points": [
                {"id": 3, "labels": [1, 0]},
                {"id": 9, "labels": [0, 1]},
                {"id": 11, "labels": [1, 1]},
            ],
        },
    ]

    for case in cases:
        dataset = pd.read_csv(
            TEST_DATASET_PATH, SEPARATOR, usecols=case["selected_columns"]
        )
        transformed_dataset = encode_and_normalize(dataset)

        if case["partition"] is not None:
            indexes_mapping = compute_indexes_mapping(
                list(range(len(case["selected_columns"]))), transformed_dataset.columns
            )
            new_partition = compute_partition_in_encoded_indexes(
                case["partition"], indexes_mapping
            )
            active_learner = FactorizedDualSpaceModel(
                KernelVersionSpace(), partition=new_partition
            )
        else:
            active_learner = KernelVersionSpace()

        exploration_manager = ExplorationManager(
            PartitionedDataset(
                transformed_dataset,
                copy=False,
            ),
            active_learner,
            subsampling=50000,
            initial_sampler=random_sampler(sample_size=3),
        )

        monkeypatch.setattr(
            src.routes.points.cache, "get", lambda key: exploration_manager
        )

        response = client.post(
            NEXT_UNLABELED_POINTS,
            data={"labeledPoints": json.dumps(case["labeled_points"])},
        )

        points_to_label = json.loads(response.data)

        assert isinstance(points_to_label, list)
        assert len(points_to_label) == 1
        assert isinstance(points_to_label[0], int)


import statistics
import random
import pandas as  pd
import plotly_express as px
import plotly.figure_factory as ff
import plotly.graph_objects as go

file1 = pd.read_csv('data.csv')
data = df['reading_time'].tolist()

dataset = []
def randomdata():
    for i in range(0,100):
        index = random.randint(0,(len(data) - 1))
        value = data[index]
        dataset.append(value)
    mean = statistics.mean(dataset)
    return mean

def showgraph(meanlist):
    datframe = meanlist
    mean = statistics.mean(dataframe)
    graph = ff.create_distplot([dataframe], ['Result'], show_hist = False)
    graph.add_trace(go.Scatter(x = [mean,mean], y = [0,1], mode = 'lines', name = 'Mean'))
    graph.show()

def setup():
    meanlist = []
    for i in range(0,1000):
        setofmeans = randomdata()
        meanlist.append(setofmeans)
    showfig(meanlist)
    mean = statistics.mean(meanlist)
    print('mean of sampling distribution' ,mean)
    sd = statistics.stdev(data)
    print(sd)

setup()


from data import *
from models import *
import argparse
import os
import pickle

parser = argparse.ArgumentParser(description='NLI training')

parser.add_argument("--data_path", type=str, default='./data', help="path to data")

# model
parser.add_argument("--encoder_type", type=str, default='GRUEncoder', help="see list of encoders")
parser.add_argument("--enc_lstm_dim", type=int, default=256, help="encoder nhid dimension")
parser.add_argument("--num_layer", type=int, default=1, help="encoder num layers")
parser.add_argument("--fc_dim", type=int, default=256, help="nhid of fc layers")
parser.add_argument("--n_classes", type=int, default=3, help="entailment/neutral/contradiction")
parser.add_argument("--pool_type", type=str, default='max', help="max or mean")
parser.add_argument("--use_cuda", type=bool, default=True, help="True or False")

# train
parser.add_argument("--n_epochs", type=int, default=20)
parser.add_argument("--batch_size", type=int, default=64)
parser.add_argument("--dpout_model", type=float, default=0., help="encoder dropout")
parser.add_argument("--dpout_fc", type=float, default=0.2, help="classifier dropout")
parser.add_argument("--dpout_embed", type=float, default=0., help="embed dropout")
parser.add_argument("--lr", type=float, default=0.001, help="learning rate for adam")
parser.add_argument("--last_model", type=str, default="", help="train on last saved model")
parser.add_argument("--saved_model_name", type=str, default="model_new", help="saved model name")
parser.add_argument("--w2v_model", type=str, default="w2v-model.txt", help="w2v file name")

params, _ = parser.parse_known_args()
print(params)

'''
SEED
'''
np.random.seed(10)
torch.manual_seed(10)

"""
DATA
"""
train, dev, test = get_nl(params.data_path)
wv, default_wv = build_vocab(np.append(train['s1'], train['s2']), params.w2v_model)

'''
MODEL
'''

config_nli_model = {
    'n_words'        :  len(wv),
    'word_emb_dim'   :  300,
    'enc_lstm_dim'   :  params.enc_lstm_dim,
    'num_layer'      :  params.num_layer,
    'dpout_model'    :  params.dpout_model,
    'dpout_fc'       :  params.dpout_fc,
    'fc_dim'         :  params.fc_dim,
    'bsize'          :  params.batch_size,
    'n_classes'      :  params.n_classes,
    'pool_type'      :  params.pool_type,
    'encoder_type'   :  params.encoder_type,
    'use_cuda'       :  params.use_cuda,
}

nli_net = NLINet(config_nli_model)
if params.last_model:
    print("load model {}".format(params.last_model))
    nli_net.load_state_dict(torch.load(os.path.join("saved_model", params.last_model)))
print(nli_net)

# loss
weight = torch.FloatTensor(3).fill_(1)
loss_fn = nn.CrossEntropyLoss(weight=weight)
loss_fn.size_average = False

# optimizer
from torch import optim
optimizer = optim.Adam(nli_net.parameters(), lr=params.lr)

# cuda
if params.use_cuda:
    torch.cuda.manual_seed(10)
    torch.cuda.set_device(0)
    nli_net.cuda()
    loss_fn.cuda()

'''
TRAIN
'''
def trainepoch(epoch):
    all_costs = []
    tot_costs = []
    logs = []
    correct = 0.0

    nli_net.train()
    permutation = np.random.permutation(len(train['s1']))
    s1 = train['s1'][permutation]
    s2 = train['s2'][permutation]
    target = train['label'][permutation]

    for stidx in range(0, len(s1), params.batch_size):
        s1_batch, s1_len = get_batch(s1[stidx:stidx+params.batch_size], wv, default_wv, params.dpout_embed)
        s2_batch, s2_len = get_batch(s2[stidx:stidx+params.batch_size], wv, default_wv, params.dpout_embed)

        if params.use_cuda:
            s1_batch, s2_batch = Variable(s1_batch.cuda()), Variable(s2_batch.cuda())
            tgt_batch = Variable(torch.LongTensor(target[stidx:stidx+params.batch_size])).cuda()
        else:
            s1_batch, s2_batch = Variable(s1_batch), Variable(s2_batch)
            tgt_batch = Variable(torch.LongTensor(target[stidx:stidx+params.batch_size]))
        k = s1_batch.size(1)
        output = nli_net((s1_batch, s1_len), (s2_batch, s2_len))

        pred = output.data.max(1)[1]
        correct += pred.long().eq(tgt_batch.data.long()).cpu().sum().item()

        # loss
        loss = loss_fn(output, tgt_batch)
        all_costs.append(loss.item())
        tot_costs.append(loss.item())

        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        if len(all_costs) == 100:
            logs.append('{0};  loss: {1};  accuracy train: {2}'.format(stidx,
                            round(np.mean(all_costs), 2), round(100.*correct/(stidx+k), 2)))
            print(logs[-1])
            all_costs = []

    train_acc = round(100 * correct/len(s1), 2)
    train_loss = round(np.mean(tot_costs), 2)
    return train_loss, train_acc

val_acc_best = -1e10
adam_stop = False
stop_training = False

def evaluate(epoch, eval_type='dev', final_eval=False):
    nli_net.eval()
    correct = 0.0
    global val_acc_best, lr, stop_training, adam_stop

    s1 = dev['s1'] if eval_type == 'dev' else test['s1']
    s2 = dev['s2'] if eval_type == 'dev' else test['s2']
    target = dev['label'] if eval_type == 'dev' else test['label']

    for i in range(0, len(s1), params.batch_size):
        # prepare batch
        s1_batch, s1_len = get_batch(s1[i:i + params.batch_size], wv, default_wv, params.dpout_embed)
        s2_batch, s2_len = get_batch(s2[i:i + params.batch_size], wv, default_wv, params.dpout_embed)

        if params.use_cuda:
            s1_batch, s2_batch = Variable(s1_batch.cuda()), Variable(s2_batch.cuda())
            tgt_batch = Variable(torch.LongTensor(target[i:i + params.batch_size])).cuda()
        else:
            s1_batch, s2_batch = Variable(s1_batch), Variable(s2_batch)
            tgt_batch = Variable(torch.LongTensor(target[i:i + params.batch_size]))

        # model forward
        output = nli_net((s1_batch, s1_len), (s2_batch, s2_len))

        pred = output.data.max(1)[1]
        correct += pred.long().eq(tgt_batch.data.long()).cpu().sum().item()

    # save model
    eval_acc = round(100 * correct / len(s1), 2)
    if final_eval:
        print('finalgrep:  accuracy {0}: {1}'.format(eval_type, eval_acc))
    else:
        print('togrep:  results: epoch {0};  mean accuracy {1}:{2}'.format(epoch, eval_type, eval_acc))

    if eval_type == 'dev' and eval_acc > val_acc_best:
        with open( os.path.join("saved_model", params.saved_model_name+"_config.pickle" ), 'wb') as handle:
            pickle.dump(params, handle, protocol=pickle.HIGHEST_PROTOCOL)
        print('saving model at epoch {0}'.format(epoch))
        if not os.path.exists("saved_model"): os.makedirs("saved_model")
        torch.save(nli_net.state_dict(), os.path.join("saved_model", params.saved_model_name))
        val_acc_best = eval_acc

    return eval_acc

"""
Train model
"""
### TRAINING

train_loss_ls = []
train_acc_ls = []
eval_acc_ls = []
eval_acc = 0
for i in range(params.n_epochs):
    print('\nTRAINING : Epoch ' + str(i))
    train_loss, train_acc = trainepoch(i)
    train_loss_ls.append(train_loss)
    train_acc_ls.append(train_acc)
    print('results: epoch {0};  loss: {1} mean accuracy train: {2}'.format(i, train_loss, train_acc))
    if i%1==0:
        print("-"*100)
        print('\nEVALIDATING: Epoch ' + str(i))
        eval_acc = evaluate(i, eval_type='dev', final_eval=False)
        eval_acc_ls.append(eval_acc)
        print('results: epoch {0};  mean accuracy dev: {1}'.format(i, eval_acc))
        print("-"*100)


# MAGIC CODEFORCES PYTHON FAST IO
import atexit
import io
import sys

_INPUT_LINES = sys.stdin.read().splitlines()
input = iter(_INPUT_LINES).__next__
_OUTPUT_BUFFER = io.StringIO()
sys.stdout = _OUTPUT_BUFFER

@atexit.register
def write():
    sys.__stdout__.write(_OUTPUT_BUFFER.getvalue())
# END OF MAGIC CODEFORCES PYTHON FAST IO

import itertools

def solve():
    n = int(input())
    a = map(int, input().split())
    b = map(int, input().split())
    delta = map(lambda x,y : x-y,a,b)
    partialSums = sorted(itertools.accumulate(delta))
    median = partialSums[n//2]
    print(sum(abs(median - x) for x in partialSums))

solve()


# -*- coding: utf-8 -*-
import pymysql

from learn_pymysql.test_api.mysql_api import MysqlClient
from learn_pymysql.test_api.redis import RedisClient
from learn_project.my_project.test_api.test_public import Job

class StuChooseCls(object):
    @staticmethod
    def create_course(token, course):
        """
        创建课程
        :param token:
        :param course:
        :return:
        """
        # # 判断token是否合法 又不用判断了
        # if not ValidCheckUtils.is_en_num(token):
        #     print("token不合法")
        #     return

        #  start_time不允许大于end_time，end_time不允许小于当前时间
        if course.end_time > course.start_time and course.end_time > Job.get_time():
            connection = MysqlClient.get_connection()
            db = connection.cursor(pymysql.cursors.DictCursor)
            user_id = RedisClient.create_redis_client().get("user_login_" + str(token))
            if user_id is None:
                print('用户未登录')
            else:
                db.execute("select * from school_user where id = %s", user_id)
                res = db.fetchone()
                db.execute("select * from school_user where id = %s", course.teach_id)
                teach_res = db.fetchone()
                # 以及判断token的权限是否为老师
                if res is None:
                    print("用户不存在")
                elif res['type'] == 0:
                    print('非教师登陆')
                # teach_id必须有效（在用户表中存在，并且是type是教师类型）
                elif teach_res is None or teach_res['type'] == 0:
                    print("teach_id不存在或权限错误")
                else:
                    #  录入课程的信息到课程表中
                    db.execute(
                        "insert into school_class(name,teach_id,count,new_count,start_time,end_time,create_time) values (%s,%s,%s,%s,%s,%s,%s)",
                        (course.name, course.teach_id, course.count, 0, course.start_time,
                         course.end_time, Job.get_time()))
                    connection.commit()
        else:
            print("课程时间参数不合法")

    @staticmethod
    def get_course():
        """
        获取课程列表
        :return:
        """
        connection = MysqlClient.get_connection()
        db = connection.cursor(pymysql.cursors.DictCursor)
        db.execute("select * from school_class")
        res = db.fetchall()
        course_list = []
        for i in res:
            course_dict = {}
            if i['start_time'] <= Job.get_time() <= i['end_time']:
                for k, v in i.items():
                    if k == 'teach_id':
                        db.execute("select * from school_user where id=%s", v)
                        teach_name = db.fetchone()['name']
                        course_dict['teach_name'] = teach_name
                    else:
                        course_dict[k] = v
            course_list.append(course_dict)
        return course_list

    @staticmethod
    def choose_course(token, course_id):
        """
         选课
        :param token:
        :param course_id:
        :return:
        """
        connection = MysqlClient.get_connection()
        db = connection.cursor(pymysql.cursors.DictCursor)
        user_id = RedisClient.create_redis_client().get("user_login_" + str(token))
        # 判断token对应的用户是否存在
        if user_id is None:
            print('用户未登录')
            return False
        else:
            db.execute("select * from school_user where id = %s", user_id)
            res = db.fetchone()
            # 判断token对应的用户是否存在
            if res is None:
                print("用户不存在")
                return False
            else:
                # 查询 course_id 的 school_class
                db.execute("select * from school_class where id = %s", course_id)
                cls_res = db.fetchone()
                # 查询course_id 的选课信息
                db.execute("select * from user_class where class_id = %s and user_id=%s", (course_id, user_id))
                user_cls_res = db.fetchone()
                # 判断用户是否选过了这门课
                if user_cls_res is not None:
                    print('用户已经选过该课程')
                    return False
                # 这门课程当前时间还能否选
                elif not (cls_res['start_time'] < Job.get_time() < cls_res['end_time']):
                    print("当前时间该课程不可选")
                    return False
                # 选课人数是否到达上限
                elif cls_res['new_count'] + 1 > cls_res['count']:
                    print("选课人数达上限")
                    return False
                else:
                    # 新增学生选课表
                    db.execute("insert into user_class(user_id, class_id, create_time) values (%s,%s,%s)",
                               (user_id, course_id, Job.get_time()))
                    connection.commit()
                    # 修改课程表数据
                    db.execute("update school_class set new_count=%s where id = %s",
                               (cls_res['new_count'] + 1, course_id))
                    connection.commit()
                    return True


from telegram import Update, Bot, InlineKeyboardButton, InlineKeyboardMarkup
from telegram.ext import Updater, CommandHandler, MessageHandler, Filters, CallbackQueryHandler
from tg_token import token

bot = Bot(token)
updater = Updater(token, use_context = True)
dispatcher = updater.dispatcher

def printt(update, context, message: str):
    context.bot.send_message(update.effective_chat.id, message)

def start(update, context):
    #context.bot.send_message(update.effective_chat.id, 'Hello')
    printt(update, context, 'Hello')

def message(update, context):
    if update.message.text == '1':
        context.bot.send_message(update.effective_chat.id, 'Goodbye')

def help_message(update, context):
    context.bot.send_message(update.effective_chat.id, 'I don`t know this command :(')

def get_day(update, context):
    keyboard =[
        [InlineKeyboardButton('Monday', callback_data = '1'), InlineKeyboardButton('Tuesday', callback_data = '2'), \
        InlineKeyboardButton('Wednesday', callback_data = '3'), InlineKeyboardButton('Thursday', callback_data = '4')],
        [InlineKeyboardButton('Friday', callback_data = '5'), InlineKeyboardButton('Saturday', callback_data = '6'), \
        InlineKeyboardButton('Sunday', callback_data = '7')]
    ]

    update.message.reply_text('Choose the day!', reply_markup = InlineKeyboardMarkup(keyboard))

def button(update, context):
    querry = update.callback_query
    querry.answer()
    if querry.data == '1': printt(update, context, 'Monday')
    elif querry.data == '2': printt(update, context, 'Tuesday')
    elif querry.data == '3': printt(update, context, 'Wednesday')
    elif querry.data == '4': printt(update, context, 'Thursday')
    elif querry.data == '5': printt(update, context, 'Friday')
    elif querry.data == '6': printt(update, context, 'Saturday')
    elif querry.data == '7': printt(update, context, 'Sunday')

buttons_handler = CallbackQueryHandler(button)
start_handler = CommandHandler('start', start) #первые обязательно команды
get_day_handler = CommandHandler('get_day', get_day)
unknown_handler = MessageHandler(Filters.command, help_message) #после unknown при наличии
message_handler = MessageHandler(Filters.text, message) #и только потом сообщения

dispatcher.add_handler(start_handler)
dispatcher.add_handler(buttons_handler)
dispatcher.add_handler(get_day_handler)
dispatcher.add_handler(unknown_handler)
dispatcher.add_handler(message_handler)

updater.start_polling()
updater.idle()


import speech_recognition as sr
import os
from gtts import gTTS
import datetime
import pyttsx3
import playsound
import warnings
import pyaudio
import random
import datetime
import time
import calendar
import wikipedia

warnings.filterwarnings("ignore")

engine = pyttsx3.init()
voices = engine.getProperty('rate')
engine.setProperty('rate', 175)

def talk(audio):  # storing audio
    engine.say(audio)
    engine.runAndWait()

def get_voice_command():  # recoding the audio with the help of google
    record = sr.Recognizer()

    with sr.Microphone() as source:
        record.adjust_for_ambient_noise(source)
        print("Listening...")
        audio = record.listen(source, timeout=10)

    data = ''

    try:
        data = record.recognize_google(audio)
    except sr.UnknownValueError:
        engine.say("Assistant could not understand the audio")
    except sr.RequestError as ex:
        engine.say("Request Error from Google Speech Recognition" + ex)
    return data

def response(text):  # Air responding to the usr
    tts = gTTS(text=text, lang="en")
    audio = "Audio.mp3"
    tts.save(audio)
    playsound.playsound(audio)
    os.remove(audio)  # removing audio as a nessesity to keep it clear

def call(text):  # calling air
    action_call = "air"
    text = text.lower()
    if action_call in text:
        return True
    return False

def today_date():
    now = datetime.datetime.now()
    date_now = datetime.datetime.today()
    week_now = calendar.day_name[date_now.weekday()]
    month_now = now.month
    day_now = now.day

    months = [
        "January",
        "February",
        "March",
        "April",
        "May",
        "June",
        "July",
        "August",
        "September",
        "October",
        "November",
        "December",
    ]

    ordinals = [
        "1st",
        "2nd",
        "3rd",
        "4th",
        "5th",
        "6th",
        "7th",
        "8th",
        "9th",
        "10th",
        "11th",
        "12th",
        "13th",
        "14th",
        "15th",
        "16th",
        "17th",
        "18th",
        "19th",
        "20th",
        "21st",
        "22nd",
        "23rd",
        "24th",
        "25th",
        "26th",
        "27th",
        "28th",
        "29th",
        "30th",
        "31st",
    ]

    return "Today is " + week_now + ", " + months[month_now - 1] + " the " + ordinals[day_now - 1]

def wiki_finder(text):
    extras = ["wikipedia", "who", "is", "search", "on", "at", " "]
    original_text = text.split()
    to_be_searched = ""
    for imp in range(0, len(original_text)):
        if original_text[imp] != ".":
            if original_text[imp] not in extras:
                to_be_searched = to_be_searched + original_text[imp]
                to_be_searched = to_be_searched + " "
        else:
            break
    return to_be_searched

while True:
    try:
        text = get_voice_command()
        speak = ""

        if "date" in text or "day" in text or "month" in text or "what do you think today's date is" in text:
            get_today = today_date()
            speak = speak + " " + get_today

        elif "time" in text:
            now = datetime.datetime.now()
            meridiem = ""
            if now.hour >= 12:
                meridiem = "pm"
                hour = now.hour - 12
            else:
                meridiem = "am"
                hour = now.hour

            if now.minute < 10:
                minute = "0" + str(now.minute)
            else:
                minute = str(now.minute)
            speak = speak + " " + "It is " + str(hour) + ":" + minute + " " + meridiem + " ."

        elif "wikipedia" in text or "Wikipedia" in text:
            object = wiki_finder(text)
            wiki = wikipedia.summary(object, sentences=2)
            speak = speak + " " + wiki
            wiki = ""

        elif "thank you" in text or "shutdown" in text:
            talk("see you again")
            break

        response(speak)
        print(speak)
    except:
        talk("I didn't understand what you said please try again")


from selenium import webdriver
from selenium.webdriver.common.keys import Keys

"""
在   main   方法中才会延迟输出
"""
def auto_search():
    """
    自动搜索
    :return:
    """
    driver = webdriver.Chrome()

    url = "http://www.dianping.com"
    # url = "https://www.baidu.com/"
    driver.get(url)

    # 获取页面元素
    print(driver.page_source)

    # 获取 cookies
    cookies = driver.get_cookies()
    print(cookies)

    # 搜索
    elem = driver.find_element_by_id('J-search-input')
    elem.clear()
    elem.send_keys("火锅")
    elem.send_keys(Keys.RETURN)

    # driver.close()

if __name__ == '__main__':
    # auto_search()
    options = webdriver.ChromeOptions()
    options.add_experimental_option('excludeSwitches', ['enable-automation'])
    driver = webdriver.Chrome(options=options)
    # driver = webdriver.Chrome()

    url = "http://www.dianping.com"
    # url = "https://www.baidu.com/"
    url = "http://www.dianping.com/shop/110281977"
    driver.get(url)

    # 获取页面元素
    all_page_source = driver.page_source
    print(all_page_source)

    print("sssssss", all_page_source.find("地址不对"))

    # 获取 cookies
    cookies = driver.get_cookies()
    print(cookies)

    # 搜索
    elem = driver.find_element_by_id('J-search-input')
    elem.clear()
    elem.send_keys("火锅")
    elem.send_keys(Keys.RETURN)

    # driver.close()


# -*- coding: utf-8 -*-
"""
Created on Tue Mar 17 15:15:36 2020

@author: Administrator
"""

import numpy as np
import torch
import torch.nn as nn
import time
#import skimage.measure as  sm
import skimage.metrics  as  sm
import cv2
from osgeo import gdal
import matplotlib.pyplot as plt

###img read tool###############################################################
def imgread(file,mode='gdal'):
    if mode=='cv2':
        img=cv2.imread(file,-1)/10000.
    if mode=='gdal':
        img=gdal.Open(file).ReadAsArray()/10000.
    return img

###weight caculate tools######################################################
def weight_caculate(data):
    return  torch.log((abs(data)*10000+1.00001))

def caculate_weight(l1m1,m1m2):
    #atmos difference
    wl1m1=weight_caculate(l1m1 )
    #time deference
    wm1m2=weight_caculate(m1m2 )
    return  wl1m1*wm1m2

###space distance caculate tool################################################
def indexdistance(window):
    #one window, one distance weight matrix
    [distx,disty]=np.meshgrid(np.arange(window[0]),np.arange(window[1]))
    centerlocx,centerlocy=(window[0]-1)//2,(window[1]-1)//2
    dist=1+(((distx-centerlocx)**2+(disty-centerlocy)**2)**0.5)/((window[0]-1)//2)
    return  dist

###threshold select tool######################################################
def weight_bythreshold(weight,data,threshold):
    #make weight tensor
    weight[data<=threshold]=1
    return  weight
def weight_bythreshold_allbands(weight,l1m1,m1m2,thresholdmax):
    #make weight tensor
    weight[l1m1<=thresholdmax[0]]=1
    weight[m1m2<=thresholdmax[1]]=1
    allweight=(weight.sum(0).view(1,weight.shape[1],weight.shape[2]))/weight.shape[0]
    allweight[allweight!=1]=0
    return  allweight

###initial similar pixels tools################################################
def spectral_similar_threshold(clusters,NIR,red):
    thresholdNIR=NIR.std()*2/clusters
    thresholdred=red.std()*2/clusters
    return  (thresholdNIR,thresholdred)

def caculate_similar(l1,threshold,window):
    #read l1
    device= torch.device("cuda" if torch.cuda.is_available() else "cpu")
    l1=nn.functional.unfold(l1,window)
    #caculate similar
    weight=torch.zeros(l1.shape,dtype=torch.float32).to(device)
    centerloc=( l1.size()[1]-1)//2
    weight=weight_bythreshold(weight,abs(l1-l1[:,centerloc:centerloc+1,:]) ,threshold)
    return weight

def classifier(l1):
    '''not used'''
    return

###similar pixels filter tools#################################################
def allband_arrayindex(arraylist,indexarray,rawindexshape):
    shape=arraylist[0].shape
    datalist=[]
    for array in arraylist:
        newarray=torch.zeros(rawindexshape,dtype=torch.float32).cuda()
        for band in range(shape[1]):
            newarray[0,band]=array[0,band][indexarray]
        datalist.append(newarray)
    return  datalist

def similar_filter(datalist,sital,sitam):
    [l1,m1,m2]=datalist
    l1m1=abs(l1-m1)
    m1m2=abs(m2-m1)
    #####
    l1m1=nn.functional.unfold(l1m1,(1,1)).max(1)[0]+(sital**2+sitam**2)**0.5
    m1m2=nn.functional.unfold(m1m2,(1,1)).max(1)[0]+(sitam**2+sitam**2)**0.5
    return (l1m1,m1m2)

###starfm for onepart##########################################################
def starfm_onepart(datalist,similar,thresholdmax,window,outshape,dist):
    #####param and data
    [l1,m1,m2]=datalist
    bandsize=l1.shape[1]
    outshape=outshape
    blocksize=outshape[0]*outshape[1]
    device= torch.device("cuda" if torch.cuda.is_available() else "cpu")
    #####img to col
    l1=nn.functional.unfold(l1,window)
    m1=nn.functional.unfold(m1,window)
    m2=nn.functional.unfold(m2,window)
    l1=l1.view(bandsize,-1,blocksize)
    m1=m1.view(bandsize,-1,blocksize)
    m2=m2.view(bandsize,-1,blocksize)
    l1m1=abs(l1-m1)
    m1m2=abs(m2-m1)
    #####caculate weights
    #time and space weight
    w=caculate_weight(l1m1,m1m2)
    w=1/(w*dist)
    #similar pixels: 1:by threshold 2:by classifier
    wmask=torch.zeros(l1.shape,dtype=torch.float32).to(device)

    #filter similar pixels  for each band: (bandsize,windowsize,blocksize)
    #wmasknew=weight_bythreshold(wmask,l1m1,thresholdmax[0])
    #wmasknew=weight_bythreshold(wmasknew,m1m2,thresholdmax[1])

    #filter similar pixels for all bands: (1,windowsize,blocksize)
    wmasknew=weight_bythreshold_allbands(wmask,l1m1,m1m2,thresholdmax)
    #mask
    w=w*wmasknew*similar
    #normili
    w=w/(w.sum(1).view(w.shape[0],1,w.shape[2]))
    #####predicte and trans
    #predicte l2
    l2=(l1+m2-m1)*w
    l2=l2.sum(1).reshape(1,bandsize,l2.shape[2])
    #col to img
    l2=nn.functional.fold(l2.view(1,-1,blocksize),outshape,(1,1))
    return l2
###starfm for allpart#########################################################
def starfm_main(l1r,m1r,m2r,
                param={'part_shape':(140,140),
               'window_size':(31,31),
               'clusters':5,
               'NIRindex':3,'redindex':2,
               'sital':0.001,'sitam':0.001}):
    #get start time
    time_start=time.time()
    device= torch.device("cuda" if torch.cuda.is_available() else "cpu")
    #read parameters
    parts_shape=param['part_shape']
    window=param['window_size']
    clusters=param['clusters']
    NIRindex=param['NIRindex']
    redindex=param['redindex']
    sital=param['sital']
    sitam=param['sitam']
    #caculate initial similar pixels threshold
    threshold=spectral_similar_threshold(clusters,l1r[:,NIRindex:NIRindex+1],l1r[:,redindex:redindex+1])
    print('similar threshold (NIR,red)',threshold)
    ####shape
    imageshape=(l1r.shape[1],l1r.shape[2],l1r.shape[3])
    print('datashape:',imageshape)
    row=imageshape[1]//parts_shape[0]+1
    col=imageshape[2]//parts_shape[1]+1
    padrow=window[0]//2
    padcol=window[1]//2
    #####padding constant for conv;STARFM use Inverse distance weight(1/w),better to avoid 0 and NAN(1/0),or you can use another distance measure
    constant1=10
    constant2=20
    constant3=30
    l1=torch.nn.functional.pad( l1r,(padrow,padcol,padrow,padcol),'constant', constant1)
    m1=torch.nn.functional.pad( m1r,(padrow,padcol,padrow,padcol),'constant', constant2)
    m2=torch.nn.functional.pad( m2r,(padrow,padcol,padrow,padcol),'constant', constant3)
    #split parts , get index and  run for every part
    row_part=np.array_split( np.arange(imageshape[1]), row , axis = 0)
    col_part=np.array_split( np.arange(imageshape[2]),  col, axis = 0)
    print('Split into {} parts,row number: {},col number: {}'.format(len(row_part)*len(row_part),len(row_part),len(row_part)))
    dist=nn.functional.unfold(torch.tensor(  indexdistance(window),dtype=torch.float32).reshape(1,1,window[0],window[1]),window).to(device)

    for rnumber,row_index in enumerate(row_part):
        for cnumber,col_index in enumerate(col_part):
            ####run for part: (rnumber,cnumber)
            print('now for part{}'.format((rnumber,cnumber)))
            ####output index
            rawindex=np.meshgrid(row_index,col_index)
            ####output shape
            rawindexshape=(col_index.shape[0],row_index.shape[0])
            ####the real parts_index ,for reading the padded data
            row_pad=np.arange(row_index[0],row_index[len(row_index)-1]+window[0])
            col_pad=np.arange(col_index[0],col_index[len(col_index)-1]+window[1])
            padindex=np.meshgrid(row_pad,col_pad)
            padindexshape=(col_pad.shape[0],row_pad.shape[0])
            ####caculate initial similar pixels
            NIR_similar=caculate_similar(l1[0,NIRindex][ padindex ].view(1,1,padindexshape[0],padindexshape[1]),threshold[0],window)
            red_similar=caculate_similar(l1[0,redindex][ padindex ].view(1,1,padindexshape[0],padindexshape[1]),threshold[1],window)
            similar=NIR_similar*red_similar
            ####caculate threshold used for similar_pixels_filter
            thresholdmax=similar_filter( allband_arrayindex([l1r,m1r,m2r],rawindex,(1,imageshape[0],rawindexshape[0],rawindexshape[1])),
                                        sital,sitam)
            ####Splicing each col at rnumber-th row
            if cnumber==0:
                rowdata=starfm_onepart( allband_arrayindex([l1,m1,m2],padindex,(1,imageshape[0],padindexshape[0],padindexshape[1])),
                                       similar,thresholdmax,window,rawindexshape,dist
                                       )

            else:
                rowdata=torch.cat( (rowdata,
                                    starfm_onepart( allband_arrayindex([l1,m1,m2],padindex,(1,imageshape[0],padindexshape[0],padindexshape[1])),
                                                   similar,thresholdmax,window,rawindexshape,dist)  ) ,2)
        ####Splicing each row
        if rnumber==0:
            l2_fake=rowdata
        else:
            l2_fake=torch.cat((l2_fake,rowdata),3)

    l2_fake=l2_fake.transpose(3,2)
    #time cost
    time_end=time.time()
    print('now over,use time {:.4f}'.format(time_end-time_start))
    return l2_fake

def test():
    ##three band datas(sorry,just find them at home,i cant recognise the spectral response range of each band,'NIR' and 'red' are only examples)
    l1file='L72000306_SZ_B432_30m.tif'
    l2file='L72002311_SZ_B432_30m.tif'
    m1file='MOD09_2000306_SZ_B214_250m.tif'
    m2file='MOD09_2002311_SZ_B214_250m.tif'

    ##param
    param={'part_shape':(75,75),
           'window_size':(31,31),
           'clusters':5,
           'NIRindex':1,'redindex':0,
           'sital':0.001,'sitam':0.001}

    ##read images from files(numpy)
    l1=imgread(l1file)
    m1=imgread(m1file)
    m2=imgread(m2file)
    l2_gt=imgread(l2file)

    ##numpy to tensor
    shape=l1.shape
    l1r=torch.tensor(l1.reshape(1,shape[0],shape[1],shape[2]) ,dtype=torch.float32)
    m1r=torch.tensor(m1.reshape(1,shape[0],shape[1],shape[2]) ,dtype=torch.float32)
    m2r=torch.tensor(m2.reshape(1,shape[0],shape[1],shape[2]) ,dtype=torch.float32)
    device= torch.device("cuda" if torch.cuda.is_available() else "cpu")
    l1r=l1r.to(device)
    m1r=m1r.to(device)
    m2r=m2r.to(device)

    ##predicte(tensor input —> tensor output)
    l2_fake=starfm_main(l1r,m1r,m2r,param)
    print(l2_fake.shape)

    ##tensor to numpy
    if device.type=='cuda':
        l2_fake=l2_fake[0].cpu().numpy()
    else:
        l2_fake=l2_fake[0].numpy()

    ##show results
    #transform:(chanel,H,W) to (H,W,chanel)
    l2_fake=l2_fake.transpose(1,2,0)
    l2_gt=l2_gt.transpose(1,2,0)
    l1=l1.transpose(1,2,0)
    m1=m1.transpose(1,2,0)
    m2=m2.transpose(1,2,0)
    #plot
    plt.figure('landsat:t1')
    plt.imshow(l1)
    plt.figure('landsat:t2_fake')
    plt.imshow(l2_fake)
    plt.figure('landsat:t2_groundtrue')
    plt.imshow(l2_gt)

    ##evaluation
    ssim1=sm.structural_similarity(l2_fake,l2_gt,data_range=1,multichannel=True)
    ssim2=sm.structural_similarity(l1,l2_gt,data_range=1,multichannel=True)
    ssim3=sm.structural_similarity(l1+m2-m1,l2_gt,data_range=1,multichannel=True)
    print('with-similarpixels ssim: {:.4f};landsat_t1 ssim: {:.4f};non-similarpixels ssim: {:.4f}'.format(ssim1,ssim2,ssim3))

    return
if __name__ == "__main__":
    test()


import torch
import torch.nn as nn
import statistics
import torchvision.models as models
import torch.nn.functional as F
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

class encoder(nn.Module):
    def __init__(self, embed_size, model, unfreeze=10):
        super(encoder, self).__init__()

        self.resnet = model
        self.resnet = nn.Sequential(*list(self.resnet.children())[:-2])

        self.total_trainable_layers = 0
        self.freeze_count = 0

        for name, param in self.resnet.named_parameters():
          if param.requires_grad:
              self.total_trainable_layers += 1
              if(self.freeze_count + unfreeze < 60):
                  param.requires_grad = False
                  self.freeze_count += 1

        print("Total trainable distil bert layers are : " + str(self.total_trainable_layers))
        print("Layers freezed = "+str(self.freeze_count))

        self.relu = nn.ReLU()
        self.dropout = nn.Dropout(0.5)

    def forward(self, images):
        features = self.resnet(images)
        features = features.permute(0, 2, 3, 1)
        features = features.view(features.size(0), -1, features.size(-1)) #(B, 7*7, 512)
        return self.dropout(self.relu(features))

class Attention(nn.Module):
    def __init__(self, encoder_dim, decoder_dim, attention_dim):
        super(Attention, self).__init__()

        self.attention_dim = attention_dim
        self.W = nn.Linear(decoder_dim, attention_dim)
        self.U = nn.Linear(encoder_dim, attention_dim)
        self.A = nn.Linear(attention_dim, 1)

        self.init_h = nn.Linear(encoder_dim, decoder_dim)

    def forward(self, features, hidden_state):
        u_hs = self.U(features)     #(batch_size,num_layers,attention_dim)
        w_ah = self.W(hidden_state) #(batch_size,attention_dim)

        combined_states = torch.tanh(u_hs + w_ah.unsqueeze(1)) #(batch_size,num_layers,attemtion_dim)

        attention_scores = self.A(combined_states)         #(batch_size,num_layers,1)
        attention_scores = attention_scores.squeeze(2)     #(batch_size,num_layers)

        alpha = F.softmax(attention_scores,dim=1)          #(batch_size,num_layers)

        attention_weights = features * alpha.unsqueeze(2)  #(batch_size,num_layers,features_dim)
        attention_weights = attention_weights.sum(dim=1)   #(batch_size,num_layers)

        return alpha, attention_weights

class decoder(nn.Module):
    def __init__(self, embed_size, hidden_size, vocab_size, num_layers, vocab, tokenizer):
        super(decoder, self).__init__()
        self.tokenizer = tokenizer
        self.encoder_dim = 512
        self.embed_dim = 512
        self.attention_dim = 256
        self.decoder_dim = 256
        self.vocab = vocab
        self.vocab_size = vocab_size

        self.attention = Attention(self.encoder_dim, self.decoder_dim, self.attention_dim)

        self.decode_step = nn.GRU(input_size=self.embed_dim + self.encoder_dim, hidden_size=self.decoder_dim, num_layers=num_layers, bidirectional=True, batch_first=True)

        self.h_lin = nn.Linear(self.encoder_dim, self.decoder_dim)
        self.c_lin = nn.Linear(self.encoder_dim, self.decoder_dim)
        self.f_beta = nn.Linear(self.decoder_dim, self.encoder_dim)
        self.sigmoid = nn.Sigmoid()
        self.fc = nn.Linear(self.decoder_dim, self.vocab_size)

    # def forward(self, features, captions):
    def forward(self, features):
        enc_attn = self.attention(features)
        print(enc_attn.size())

        return enc_attn

class decoder1(nn.Module):
    def __init__(self, embed_size, vocab_size, num_layers, vocab,tokenizer, max_len = 32, unfreeze = 10):
        super(decoder1, self).__init__()
        self.vocab_size = vocab_size
        self.embed_size = embed_size
        self.max_len = max_len
        self.gru = nn.GRU(input_size=512, hidden_size=512, num_layers=2, bidirectional=True, batch_first=True)
        self.distil_bert = DistilBertModel.from_pretrained("distilbert-base-uncased",).to(device=device)
        self.tokenizer = tokenizer
        self.total_trainable_layers_distil_bert = []
        self.freeze_count = 0

        for name, param in self.distil_bert.named_parameters():
          if param.requires_grad:
              self.total_trainable_layers_distil_bert.append(name)

              if(self.freeze_count + unfreeze < 100):
                  param.requires_grad = False
                  self.freeze_count += 1

        print("Total trainable distil bert layers are : " + str(len(self.total_trainable_layers_distil_bert)))
        print("Layers freezed = "+str(self.freeze_count))

        self.bert_to_enc_dim = nn.Linear(768, 512)
        self.l1 = nn.Linear(2*512, self.vocab_size)

    def forward(self, enc_embed, ids, mask):

        embed = self.distil_bert(ids, attention_mask = mask)

        embed = embed[0]
        embed = self.bert_to_enc_dim(embed)

        all_words = []

        for i in range(self.max_len):
            cur_tens = torch.tensor([i]).to(device=device)
            word_embed = torch.index_select(embed,dim=1, index=cur_tens)

            enc_cat = torch.cat([word_embed, enc_embed], dim=1)

            e_cat, _ = self.gru(enc_cat)

            e_cat = e_cat.max(dim = 1)[0]
            e_cat = torch.squeeze(e_cat)

            e_cat = F.leaky_relu(self.l1(e_cat))
            e_cat = F.softmax(e_cat, dim=1)
            # max_prob = torch.argmax(e_cat, dim=1)

            all_words.append(e_cat)

        final_words = torch.stack(all_words, dim=2)   ## index of words
        return final_words

    def evaluate_test(self, enc_embed, ids, mask, i):

        if(i==0):
            embed = self.distil_bert(ids, attention_mask = mask)
            embed = embed[0]
            embed = self.bert_to_enc_dim(embed)
            cur_tens = torch.tensor([i]).to(device=device)
            word_embed = torch.index_select(embed,dim=1, index=cur_tens)

        else:

            embed = self.distil_bert(ids, attention_mask = mask)
            embed = embed[0]
            word_embed = self.bert_to_enc_dim(embed)

        enc_cat = torch.cat([word_embed, enc_embed], dim=1)
        e_cat, _ = self.gru(enc_cat)

        e_cat = e_cat.max(dim = 1)[0]
        e_cat = torch.squeeze(e_cat)

        e_cat = F.leaky_relu(self.l1(e_cat))
        e_cat = F.softmax(e_cat, dim=1)

        return e_cat    ## single word softmax

class fullModel(nn.Module):
    def __init__(self, embed_size, resnet, vocab_size, num_layers, vocab, tokenizer, max_len, unfreeze1, unfreeze2, inverse_vocab, vocab_list):
        super(fullModel, self).__init__()
        self.inverse_vocab = inverse_vocab
        self.vocab_size = vocab_size
        self.vocab_list = vocab_list
        self.embed_size = embed_size
        self.max_len = max_len
        self.vocab = vocab
        self.tokenizer = tokenizer

        self.enc = encoder(embed_size, resnet, unfreeze=unfreeze1)
        self.dec = decoder1(embed_size, vocab_size, num_layers, vocab, tokenizer, max_len = max_len, unfreeze = unfreeze2)

    def forward(self, image, ids, mask, test = False):

        if(not test):
            enc_embed = self.enc(image)
            words_probability = self.dec(enc_embed, ids, mask)
            words_pred = torch.argmax(words_probability, dim=1)
            captions_strings = self.index_to_captions(words_pred, mask)

            return words_probability, captions_strings

        else:
            enc_embed = self.enc(image)
            distinct_words = []
            total_probability = []
            prev_word = None

            for i in range(self.max_len):

              if(prev_word is not None):
                mask = torch.ones(ids.size(0),1)
                for i in range(len(ids)):
                  ids[i] = torch.tensor(self.vocab_list[int(prev_word[i])])

              words_probability = self.dec.evaluate_test(enc_embed, ids, mask, i)
              words_pred = torch.argmax(words_probability, dim=1)
              prev_wrod = words_pred

              distinct_words.append(words_pred)
              total_probability.append(words_probability)

            words_probability = torch.stack(total_probability, dim=2)
            all_words = torch.stack(distinct_words, dim=1)

            captions_strings = self.index_to_captions(all_words, mask)
            return words_probability, captions_strings

    def index_to_captions(self, words_tensor, mask):
        #(batch, words)
        captions = []

        for i in range(len(words_tensor)):
            cur_words = []
            ind = mask[i].argmin()
            for j in range(ind):
                cur_words.append(self.inverse_vocab[int(self.vocab_list[int(words_tensor[i][j])])])

            captions.append(' '.join(cur_words))

        return captions

class EncoderCNN(nn.Module):
    def __init__(self):
        super(EncoderCNN, self).__init__()
        resnet = models.resnet50(pretrained=True)
        for param in resnet.parameters():
            param.requires_grad_(False)

        modules = list(resnet.children())[:-2]
        self.resnet = nn.Sequential(*modules)

    def forward(self, images):
        features = self.resnet(images)                                    #(batch_size,2048,7,7)
        features = features.permute(0, 2, 3, 1)                           #(batch_size,7,7,2048)
        features = features.view(features.size(0), -1, features.size(-1)) #(batch_size,49,2048)
        return features

#Bahdanau Attention
class Attention(nn.Module):
    def __init__(self, encoder_dim,decoder_dim,attention_dim):
        super(Attention, self).__init__()

        self.attention_dim = attention_dim

        self.W = nn.Linear(decoder_dim,attention_dim)
        self.U = nn.Linear(encoder_dim,attention_dim)

        self.A = nn.Linear(attention_dim,1)

    def forward(self, features, hidden_state):
        u_hs = self.U(features)     #(batch_size,num_layers,attention_dim)
        w_ah = self.W(hidden_state) #(batch_size,attention_dim)

        combined_states = torch.tanh(u_hs + w_ah.unsqueeze(1)) #(batch_size,num_layers,attemtion_dim)

        attention_scores = self.A(combined_states)         #(batch_size,num_layers,1)
        attention_scores = attention_scores.squeeze(2)     #(batch_size,num_layers)

        alpha = F.softmax(attention_scores,dim=1)          #(batch_size,num_layers)

        attention_weights = features * alpha.unsqueeze(2)  #(batch_size,num_layers,features_dim)
        attention_weights = attention_weights.sum(dim=1)   #(batch_size,num_layers)

        return alpha,attention_weights

#Attention Decoder
class DecoderRNN(nn.Module):
    def __init__(self,embed_size, vocab_size, attention_dim,encoder_dim,decoder_dim,drop_prob=0.3):
        super().__init__()

        self.vocab_size = vocab_size
        self.attention_dim = attention_dim
        self.decoder_dim = decoder_dim

        self.embedding = nn.Embedding(vocab_size,embed_size)
        self.attention = Attention(encoder_dim,decoder_dim,attention_dim)

        self.init_h = nn.Linear(encoder_dim, decoder_dim)
        self.init_c = nn.Linear(encoder_dim, decoder_dim)
        self.lstm_cell = nn.LSTMCell(embed_size+encoder_dim,decoder_dim,bias=True)
        self.f_beta = nn.Linear(decoder_dim, encoder_dim)

        self.fcn = nn.Linear(decoder_dim,vocab_size)
        self.drop = nn.Dropout(drop_prob)

    def forward(self, features, captions):
        embeds = self.embedding(captions)
        h, c = self.init_hidden_state(features)  # (batch_size, decoder_dim)
        seq_length = len(captions[0])-1 #Exclude the last one
        batch_size = captions.size(0)
        num_features = features.size(1)

        preds = torch.zeros(batch_size, seq_length, self.vocab_size).to(device)
        alphas = torch.zeros(batch_size, seq_length,num_features).to(device)

        for s in range(seq_length):
            alpha,context = self.attention(features, h)
            lstm_input = torch.cat((embeds[:, s], context), dim=1)
            h, c = self.lstm_cell(lstm_input, (h, c))

            output = self.fcn(self.drop(h))

            preds[:,s] = output
            alphas[:,s] = alpha

        return preds, alphas

    def generate_caption(self,features,max_len=20,vocab=None):

        batch_size = features.size(0)
        h, c = self.init_hidden_state(features)  # (batch_size, decoder_dim)

        alphas = []
        word = torch.tensor(vocab.stoi['<SOS>']).view(1,-1).to(device)
        embeds = self.embedding(word)

        captions = []

        for i in range(max_len):
            alpha,context = self.attention(features, h)
            alphas.append(alpha.cpu().detach().numpy())

            lstm_input = torch.cat((embeds[:, 0], context), dim=1)
            h, c = self.lstm_cell(lstm_input, (h, c))
            output = self.fcn(self.drop(h))
            output = output.view(batch_size,-1)

            predicted_word_idx = output.argmax(dim=1)
            captions.append(predicted_word_idx.item())
            if vocab.itos[predicted_word_idx.item()] == "<EOS>":
                break
            embeds = self.embedding(predicted_word_idx.unsqueeze(0))
        return [vocab.itos[idx] for idx in captions],alphas

    def init_hidden_state(self, encoder_out):
        mean_encoder_out = encoder_out.mean(dim=1)
        h = self.init_h(mean_encoder_out)  # (batch_size, decoder_dim)
        c = self.init_c(mean_encoder_out)
        return h, c

class EncoderDecoder(nn.Module):
    def __init__(self,embed_size, vocab_size, attention_dim,encoder_dim,decoder_dim,drop_prob=0.3):
        super().__init__()
        self.encoder = EncoderCNN()
        self.decoder = DecoderRNN(
            embed_size=embed_size,
            vocab_size = vocab_size,
            attention_dim=attention_dim,
            encoder_dim=encoder_dim,
            decoder_dim=decoder_dim
        )

    def forward(self, images, captions):
        features = self.encoder(images)
        outputs = self.decoder(features, captions)
        return outputs


#!/usr/bin/env python3

import os
import argparse
import copy
import time
import json
import signal
import types
from datetime import datetime
from contextlib import contextmanager

import wandb
import yaml

def main():
    args = parse_args()
    init_wandb(args)
    log_parser = LogParser(args.experiment_dir, wait_for_new_lines=args.wait_for_logs)
    for step, log_data in log_parser.main_loop():
        wandb.log(log_data, step=step)

    print('Parser stopped')
    save_best_models(args.experiment_dir)

def parse_args():
    parser = argparse.ArgumentParser(description='Weights and Biases runner')
    parser.add_argument(
        '--experiment-dir', '-d',
        type=str,
        help='Folder with experiment',
    )
    parser.add_argument(
        '--project-name',
        default='multitarget-mt',
        help='Project name to save at wandb',
    )
    parser.add_argument(
        '--run-name',
        default=None,
        help='Project name to save at wandb',
    )
    parser.add_argument(
        '--tags', '-t',
        type=lambda s: s.split(','),
        default=[],
        help='Tag of the experiment: e.g. random, mono, wals, etc.'
    )
    parser.add_argument(
        '--infer-language-tags', '-l',
        action='store_const',
        const=True,
        default=False,
        help='Infer languages from model name, save as tags'
    )
    parser.add_argument(
        '--wait-for-logs', '-w',
        action='store_const',
        const=True,
        default=False,
        help='Wait for new lines in logs (use during the training)'
    )

    args = parser.parse_args()
    return args

def init_wandb(args):
    config = get_config(args.experiment_dir)
    run_id = get_run_id(args.experiment_dir)
    run_name = os.path.basename(os.path.normpath(args.experiment_dir))
    language_tags = get_language_tags(args)
    tags = args.tags + language_tags
    config = add_language_info(config, language_tags)
    wandb.init(
        project=args.project_name,
        name=run_name,
        resume=run_id,
        dir=args.experiment_dir,
        tags=tags,
        config=config,
    )
    save_run_id(args.experiment_dir)

def get_run_id(experiment_dir):
    try:
        with open(os.path.join(experiment_dir, '.run_id'), encoding='utf-8') as f:
            run_id = f.readline()
        return run_id
    except FileNotFoundError:
        return False

def save_run_id(experiment_dir):
    with open(os.path.join(experiment_dir, '.run_id'), 'w', encoding='utf-8') as f:
        f.write(str(wandb.run.id))

def get_language_tags(args):
    if args.infer_language_tags:
        return extract_langs(args.experiment_dir)
    else:
        return []

def extract_langs(experiment_dir):
    """ model_en2fres -> source:en, target:fr, target:es """
    useful_part = os.path.basename(os.path.normpath(experiment_dir)).split('_')[-1]
    source, target = useful_part.split('2')
    get_langs = lambda ls: [ls[i:i + 2] for i in range(0, len(ls), 2)]
    source_tags = ['source:'+tag for tag in get_langs(source)]
    target_tags = ['target:'+tag for tag in get_langs(target)]
    n_tags = [
        'n_sources:'+str(len(source_tags)),
        'n_targets:'+str(len(target_tags)),
    ]

    return source_tags + target_tags + n_tags

def add_language_info(config, language_tags):
    new_config = copy.deepcopy(config)
    n_targets = sum('target:' in tag for tag in language_tags)
    new_config['n_targets'] = n_targets if n_targets else None

    return new_config

def get_config(path):
    config_path = os.path.join(path, 'model.npz.yml')
    config = get_config_when_created(config_path)
    config = filter_config(config)
    config = fix_config_values(config)
    config = convert_config_list_values(config)
    #config = set_languages(config, path)

    return config

def get_config_when_created(path):
    def safe_load(f):
        time.sleep(2)
        return yaml.safe_load(f)

    config = read_when_created(path, safe_load, wait=10)
    print(config)
    return config

def filter_config(config):
    ignored_keys = {
        'log-level', 'quiet', 'quiet-translation',
        'train-sets', 'vocabs', 'overwrite', 'no-reload',
        'keep-best', 'valid-sets', 'log', 'valid-log',
        'relative-paths', 'model', 'ignore-model-config',
        'valid-script-path', 'tempdir',
    }

    filtered_config = {
        key: value
        for key, value
        in config.items()
        if key not in ignored_keys
    }

    return filtered_config

def fix_config_values(config):
    # change devices enumeration to device number
    fixed_config = copy.deepcopy(config)
    fixed_config['devices'] = len(fixed_config['devices'])
    # save only version, drop hash and compile date
    if 'version' in fixed_config:
        # sometimes the last line with the version is not
        # in the config immediately, but it is not an important
        # value to be stored
        fixed_config['version'] = fixed_config['version'].split()[0]

    return fixed_config

def convert_config_list_values(config):
    # replace key:[value1,value2] with key-1:value1, key-2: value2
    fixed_config = {}
    for key, value in config.items():
        if type(value) is list:
            for i,v in enumerate(value, 1):
                fixed_config[key+'-'+str(i)] = v
        else:
            fixed_config[key] = value

    return fixed_config

def set_languages(config, path):
    model_folder = os.path.basename(os.path.normpath(path))
    ls = model_folder.split('2')[-1]
    languages = [ls[i:i + 2] for i in range(0, len(ls), 2)]

    new_config = copy.deepcopy(config)

    new_config['languages'] = languages
    new_config['n_languages'] = len(languages)
    return new_config

def read_when_created(path, fn, mode='r', wait=2, stop=None):
    if stop is None:
        stop = lambda: False
    while not stop():
        try:
            with open(path, mode, encoding='utf-8') as f:
                return fn(f)
        except FileNotFoundError:
            time.sleep(wait)
            continue

def read_when_created_gen(path, fn, mode='r', wait=5, stop=None):
    if stop is None:
        stop = lambda: False
    while not stop():
        try:
            with open(path, mode, encoding='utf-8') as f:
                yield from fn(f)
            break
        except FileNotFoundError:
            time.sleep(wait)
            continue

def follow(thefile, stop=None):
    if stop is None:
        stop = lambda: False
    while True:
        line = thefile.readline()
        if not line:
            time.sleep(0.1)
            # read till the end but do not wait if stop() fired
            if stop(): break
            else: continue
        yield line

def save_best_models(experiment_dir):
    print('Saving models... ', end='')
    wandb.save(os.path.join(experiment_dir, 'model.npz.best-*'))
    wandb.save(os.path.join(experiment_dir, '*.log'))
    wandb.save(os.path.join(experiment_dir, 'model.npz.yml'))
    print('Done.')

class LogParser:
    def __init__(self, experiment_dir, wait_for_new_lines=False):
        self.experiment_dir = experiment_dir
        self.train_log_path = os.path.join(self.experiment_dir, 'train.log')
        self.wait_for_new_lines = wait_for_new_lines
        self.should_be_stopped = False

    def stop(self, signum, frame):
        print('Stopping log parser...')
        self.should_be_stopped = True

    def main_loop(self):
        """yields (step, log_dict)"""
        with signal_handler(signal.SIGINT, self.stop),\
             signal_handler(signal.SIGTERM, self.stop):
            yield from read_when_created_gen(
                self.train_log_path,
                fn=lambda f: self._process_train_log_file(f),
                stop=lambda: self.should_be_stopped,
            )

    def _process_train_log_file(self, train_log_file):
        # TODO: refactor this monster
        def extract_time(log_line):
            '''[dtime] remaining log line'''
            dtime = datetime.strptime(log_line[1:20], "%Y-%m-%d %H:%M:%S")
            return dtime, log_line[22:]

        def extract_is_validation(log_line):
            '''
            [valid] remaining_line -> True, remaining_line
            log_line -> False, log_line
            '''
            validation = '[valid]'
            if log_line[:len(validation)] == validation:
                return True, log_line[len(validation):]
            else: return False, log_line

        def is_training_log(log_line):
            return log_line.find('Ep. ') >= 0

        def parse_train_log(line):
            """ Ep. 2 : Up. 1000 : Sen. 311,824 : Cost 4.95877838 : Time 163.05s : 33655.40 words/s : L.r. 1.0000e-04 """
            processing = {
                "Ep.": lambda s: ('train/epoch', int(s)),
                "Up.": lambda s: ('step', int(s)),
                "Sen.": lambda s: ('train/sentences', int(s.replace(',',''))),
                "Cost": lambda s: ('train/loss', float(s)),
                "Time": lambda s: ('train/time', float(s[:-1])),
                "words/s": lambda s: ('train/speed', float(s)),
                "L.r.": lambda s: ('train/learning_rate', float(s)),
            }
            def process(log_line_chunk):
                first, second = log_line_chunk.strip().split(' ')
                try:
                    return processing[first](second)
                except KeyError:
                    try:
                        return processing[second](first)
                    except KeyError:
                        return None
            log_data = dict(
                process(info_chunk)
                for info_chunk
                in line.split(':')
                if info_chunk is not None
            )
            step = log_data['step']
            del log_data['step']
            return step, log_data

        def parse_val_log(line):
            """ Ep. 26 : Up. 35000 : ce-mean-words : 1.63575 : new best """
            line = line.split(':')
            step = int(line[1].strip().split(' ')[1])
            log_data = {}
            metric_name = line[2].strip()
            metric_value = float(line[3].strip())
            if 'lang/' not in metric_name:
                metric_name = 'valid/' + metric_name
            log_data[metric_name] = metric_value

            metric_stalled = line[4].strip()
            if 'no effect' not in metric_stalled:
                if 'stalled' in metric_stalled:
                    metric_stalled = int(metric_stalled.split()[1])
                else:
                    metric_stalled = 0
                log_data[metric_name+'_stalled'] = metric_stalled

            return step, log_data

        def parse_translation(lines):
            data = []
            log_data = {}
            for line in lines:
                if translation_ends(line):
                    log_data['valid/translation_time'] = float(line.strip().split(' ')[-1][:-1])
                    break
                if not 'Best translation' in line:
                    return None
                # cut time
                line = line[line.find(']')+1:]
                colon = line.find(':')
                n = line[:colon]
                translation = line[colon+1:]
                n = int(n.strip().split(' ')[-1])
                translation = translation.strip()
                data.append([n, translation])
            log_data['valid/translation_example'] = wandb.Table(
                    data=data, columns=["N","Translation"]
            )

            return log_data

        def training_finished(line):
            return "Training finished" in line

        def translation_begins(line):
            return "Translating" in line

        def translation_ends(line):
            return "Total translation time" in line

        last_step=0
        lines = follow(
            train_log_file,
            stop=lambda: self.should_be_stopped or not self.wait_for_new_lines
        )
        for line in lines:

            try:
                dtime, line = extract_time(line)
            except:
                continue

            if training_finished(line):
                break

            is_validation, line = extract_is_validation(line)

            if is_validation:
                step, log_data = parse_val_log(line)
            elif is_training_log(line):
                step, log_data = parse_train_log(line)
            elif translation_begins(line):
                # pass generator here to consume all lines with translations
                log_data = parse_translation(lines)
                # translation does not have 'step', that is why there is this 'last_step' thing
                step = last_step
            else:
                continue

            last_step = step

            yield step, log_data

@contextmanager
def signal_handler(handled_signal, new_handler):
    old_handler = signal.getsignal(handled_signal)
    try:
        signal.signal(handled_signal, new_handler)
        yield
    finally:
        signal.signal(handled_signal, old_handler)

if __name__=='__main__':
    main()


# coding:utf-8
"""
@file: .py
@author: dannyXSC
@ide: PyCharm
@createTime: 2022年05月15日 19点47分
@Function: 把协作关系转化为csv文件
"""
import pandas as pd

from Reader.CoauthorReader import read_coauthor
from utils import get_coauthor_csv_path, timer

coauthor_path = get_coauthor_csv_path()

collaboration_list = []

@timer("读取协作关系")
def read_coauthor_data():
    cnt = 0
    for collaboration in read_coauthor():
        if len(collaboration.name1.strip()) == 0 \
                or len(collaboration.name2.strip()) == 0 \
                or collaboration.count is None:
            continue
        collaboration_list.append(collaboration.to_dict())

        cnt += 1
        if cnt % 100 == 0:
            print(f"已读取{cnt}条数据")

@timer("写入csv文件")
def write_csv():
    pd.DataFrame(collaboration_list).to_csv(coauthor_path, index=False, sep=';')

def main():
    read_coauthor_data()
    write_csv()


import requests
import pandas as pd

def get_children_leis(targetLei):
    """
    Gets a list of LEIs for all ultimate children of a given LEI.

    Parameters
    ----------
    targetLei : str
        The LEI of the parent entity for which the ultimate children LEIs are to be retrieved.

    Returns
    -------
    df : DataFrame
        A Pandas DataFrame containing the LEI values for all ultimate children of the parent entity.

    """
    # Create the API URL to retrieve the list of ultimate children for the parent entity
    url = "https://api.gleif.org/api/v1/lei-records/" + targetLei + "/ultimate-children?page[size]=50&page[number]=1"

    # Get the first page of the response to extract the last page number
    response = requests.request("GET", url, headers={'Accept': 'application/vnd.api+json'}, data={})

    # Extract the JSON data from the response
    data = response.json()
    # Extract the last page number from the JSON data
    lastPage = data['meta']['pagination']['lastPage']

    df_list = []
    for currentPage in range(1, lastPage + 1):
        print('Processing page ' + str(currentPage) + ' of ' + str(lastPage))
        url = url[0:-1] + str(currentPage)
        response = requests.request("GET", url, headers={'Accept': 'application/vnd.api+json'}, data={})

        # Extract the JSON data from the response
        data = response.json()

        # Create an empty list to store the LEIs of the ultimate children
        ultimate_children_leis = []

        # Iterate over the JSON data to extract the LEI values for each ultimate child entity
        for child in data['data']:
            ultimate_children_leis.append(child['attributes']['lei'])

        # Create a Pandas DataFrame from the list of ultimate children LEIs
        df_list.append(pd.DataFrame({'LEI': ultimate_children_leis}))

    # Concatenate the DataFrames into a single DataFrame
    df = pd.concat(df_list)
    return df

targetLei = '5493006QMFDDMYWIAM13'
get_children_leis(targetLei)


import random
import uuid
import copy
import json
import multihash

from intergov.domain.jurisdiction import Jurisdiction
from intergov.domain.wire_protocols import generic_discrete as gd
from intergov.domain import uri as u
from intergov.serializers import generic_discrete_message as ser

def _random_multihash():
    digest = str(uuid.uuid4()).encode()
    return multihash.to_b58_string(multihash.encode(digest, 'sha2-256'))

def _generate_msg_dict(**kwargs):
    # test predicates must have some dots in them,
    # so that we can test the predicate__wild filter
    # tmp = str(uuid.uuid4())[:-4]
    # predicate = str("http://predicate/{}.{}.{}.{}".format(
    #         tmp[0:5],
    #         tmp[5:10],
    #         tmp[10:20],
    #         tmp[20:]))
    short_countries_list = ['AU', 'CN']
    sender = random.choice(short_countries_list)
    short_countries_list.remove(sender)
    receiver = random.choice(short_countries_list)
    predicate_parts = []
    for i in range(4):
        predicate_parts.append(_random_multihash()[:4])
    predicate = ".".join(predicate_parts)
    subject = _random_multihash()
    obj = _random_multihash()
    return {
        "sender": sender,
        "receiver": receiver,
        "subject": subject,
        "obj": obj,
        "predicate": predicate,
        **kwargs
    }

def _generate_msg_object(**kwargs):
    return gd.Message.from_dict(_generate_msg_dict(**kwargs))

def _remove_message_params(data, keys=[], copy_data=True, set_none=False):
    if copy_data:
        data = copy.deepcopy(data)
    for key in keys:
        if set_none:
            data[key] = None
        else:
            del data[key]
    return data

def _remove_required_message_params(data, index=-1, indexes=[]):
    if index >= 0:
        indexes.append(index)
    return _remove_message_params(data, keys=[gd.Message.required_attrs[i] for i in indexes])

def _encode_message_dict(data):
    return json.dumps(data, cls=ser.MessageJSONEncoder)

def _diff_message_dicts(left, right, keys=[]):
    diff = []
    for key in keys:
        l_val = left.get(key)
        r_val = right.get(key)
        if l_val != r_val:
            diff.append({
                'key': key,
                'left': l_val,
                'right': r_val
            })
    return diff

def _generate_message_params():
    msg_dict = _generate_msg_dict()

    A = Jurisdiction(msg_dict["sender"])
    B = Jurisdiction(msg_dict["receiver"])
    subject = u.URI(msg_dict["subject"])
    obj = u.URI(msg_dict["obj"])
    predicate = u.URI(msg_dict["predicate"])

    return (A, B, subject, obj, predicate)

def _generate_invalid_uri_list():
    return (None, "invalid", 42)

def test_message_from_dict():
    adict = _generate_msg_dict()
    msg = gd.Message.from_dict(adict)
    assert msg.is_valid()

def test_message_to_dict():
    adict = _generate_msg_dict()
    msg = gd.Message.from_dict(adict)
    assert msg.to_dict() == adict

def test_message_comparison():
    adict = _generate_msg_dict()
    m1 = gd.Message.from_dict(adict)
    m2 = gd.Message.from_dict(adict)
    assert m1 == m2

def test_validation_OK():
    A, B, subject, obj, predicate = _generate_message_params()
    msg = gd.Message(
        sender=A,
        receiver=B,
        subject=subject,
        obj=obj,
        predicate=predicate)
    assert msg.is_valid()

def test_validation_invalid_no_sender():
    A, B, subject, obj, predicate = _generate_message_params()
    msg = gd.Message(
        receiver=B,
        subject=subject,
        obj=obj,
        predicate=predicate)
    assert not msg.is_valid()

def test_validation_invalid_sender():
    A, B, subject, obj, predicate = _generate_message_params()
    for x in _generate_invalid_uri_list():
        A = x
        msg = gd.Message(
            sender=A,
            receiver=B,
            subject=subject,
            obj=obj,
            predicate=predicate)
        assert not msg.is_valid()

def test_validation_invalid_no_reciever():
    A, B, subject, obj, predicate = _generate_message_params()
    msg = gd.Message(
        sender=A,
        subject=subject,
        obj=obj,
        predicate=predicate)
    assert not msg.is_valid()

def test_validation_invalid_receiver():
    A, B, subject, obj, predicate = _generate_message_params()
    for x in _generate_invalid_uri_list():
        B = x
        msg = gd.Message(
            sender=A,
            receiver=B,
            subject=subject,
            obj=obj,
            predicate=predicate)
        assert not msg.is_valid()

def test_validation_invalid_no_subject():
    A, B, subject, obj, predicate = _generate_message_params()
    msg = gd.Message(
        sender=A,
        receiver=B,
        obj=obj,
        predicate=predicate)
    assert not msg.is_valid()

def test_validation_invalid_no_obj():
    A, B, subject, obj, predicate = _generate_message_params()
    msg = gd.Message(
        sender=A,
        receiver=B,
        subject=subject,
        predicate=predicate)
    assert not msg.is_valid()

def test_validation_invalid_obj():
    A, B, subject, obj, predicate = _generate_message_params()
    for x in _generate_invalid_uri_list():
        obj = x
        msg = gd.Message(
            sender=A,
            receiver=B,
            subject=subject,
            obj=obj,
            predicate=predicate)
        assert not msg.is_valid()

def test_validation_invalid_no_predicate():
    A, B, subject, obj, predicate = _generate_message_params()
    msg = gd.Message(
        sender=A,
        receiver=B,
        subject=subject,
        obj=obj)
    assert not msg.is_valid()

def test_validation_invalid_predicate():
    A, B, subject, obj, predicate = _generate_message_params()
    for x in _generate_invalid_uri_list():
        predicate = x
        msg = gd.Message(
            sender=A,
            receiver=B,
            subject=subject,
            obj=obj,
            predicate=predicate)
        assert not msg.is_valid()

def test_validation_bogus_parameter():
    A, B, subject, obj, predicate = _generate_message_params()
    msg = gd.Message(
        bogus=True,
        sender=A,
        receiver=B,
        subject=subject,
        obj=obj)
    assert not msg.is_valid()