from itertools import cycle

import torch
import zounds
from torch import nn
from torch.nn import functional as F
from torch.optim import Adam

from featuresynth.data import DataStore, MdctDataStore
from featuresynth.feature import total_samples, frequency_recomposition, \
    band_sizes, filter_banks, slices, bandpass_filters, sr, feature_channels
from featuresynth.featurediscriminator import SpecDiscriminator as Discriminator
from featuresynth.featuregenerator import SpecGenerator as Generator, AutoEncoder
from featuresynth.util import device
from featuresynth.generator import Generator as AudioGenerator

import numpy as np

batch_size = 4
n_frames = 256
total_feature_samples = total_samples * (n_frames // 64)
noise_dim = 128

audio_generator_input_size = 64
audio_generator = AudioGenerator(
    input_size=audio_generator_input_size,
    in_channels=feature_channels,
    channels=128,
    output_sizes=band_sizes,
    filter_banks=filter_banks,
    slices=slices,
    bandpass_filters=bandpass_filters)
audio_generator.load_state_dict(torch.load('generator_dilated.dat'))
[item.fb[0].to(torch.device('cpu')) for item in audio_generator.generators]

def preview(fake_batch):

    # fake_batch = fake_batch * spec_std
    # fake_batch = fake_batch + spec_mean

    fake_batch = torch.from_numpy(fake_batch)
    inp = fake_batch[0]  # (256, 512)

    window = audio_generator_input_size
    inp = inp.unfold(1, window, window)  # (256, 8, 64)

    inp = inp.permute(1, 0, 2)  # (8, 256, 64)
    bands = audio_generator(inp)
    samples = frequency_recomposition(
        [np.concatenate(b.data.cpu().numpy().reshape(1, 1, -1), axis=-1)
         for b in bands.values()],
        total_feature_samples)

    # synth = zounds.MDCTSynthesizer()
    # coeffs = zounds.ArrayWithUnits(fake_batch[0].T, [
    #     zounds.TimeDimension(frequency=sr.frequency * 256, duration=sr.frequency * 512),
    #     zounds.IdentityDimension()
    # ])
    # samples = synth.synthesize(coeffs)

    return zounds.AudioSamples(samples.squeeze(), sr).pad_with_silence()

if __name__ == '__main__':

    app = zounds.ZoundsApp(globals=globals(), locals=locals())
    app.start_in_thread()

    ds = DataStore('feature_data', '/hdd/musicnet/train_data')
    # ds = MdctDataStore('mdct_data', '/hdd/musicnet/train_data')

    def autoregressive_generation(real=True):
        if real:
            primer = next(ds.batch_stream(1, 256))
            primer = primer / (primer.max() + 1e-8)
            primer = torch.from_numpy(primer).to(device)
        else:
            primer = torch.FloatTensor(1, feature_channels, n_frames).uniform_(0, 1).to(device)
        frames = g.generate(primer, 256)
        frames = frames.data.cpu().numpy()
        return preview(frames)

    ae = AutoEncoder().initialize_weights().to(device)
    # ae.load_state_dict(torch.load('feature_autoencoder.dat'))
    ae_optim = Adam(ae.parameters(), lr=0.0001, betas=(0, 0.9))

    g = Generator(
        frames=n_frames,
        out_channels=feature_channels,
        noise_dim=128,
        initial_dim=16,
        channels=128,
        ae=ae).initialize_weights().to(device)
    g.spec.load_state_dict(torch.load('gen_spec.dat'))
    g_frame_optim = Adam(g.spec.parameters(), lr=0.0001, betas=(0, 0.9))
    g_time_optim = Adam(
        g.to_time_series.parameters(), lr=0.0001, betas=(0, 0.9))
    # g_optim = Adam(g.parameters(), lr=0.00005, betas=(0, 0.9))

    d = Discriminator(
        frames=n_frames,
        feature_channels=feature_channels,
        channels=128,
        n_judgements=1,
        ae=ae).initialize_weights().to(device)
    # d.spec_judge.load_state_dict(torch.load('disc_spec.dat'))
    d_frame_optim = Adam(d.spec_judge.parameters(), lr=0.0001, betas=(0, 0.9))
    d_time_optim = Adam(d.parameters(), lr=0.0001, betas=(0, 0.9))
    # d_optim = Adam(d.parameters(), lr=0.0001, betas=(0, 0.9))

    def zero_grad():
        g_frame_optim.zero_grad()
        g_time_optim.zero_grad()
        d_frame_optim.zero_grad()
        d_time_optim.zero_grad()
        # g_optim.zero_grad()
        # d_optim.zero_grad()
        g.zero_grad()
        d.zero_grad()

    def set_requires_grad(x, requires_grad):
        if isinstance(x, nn.Module):
            x = [x]
        for item in x:
            for p in item.parameters():
                p.requires_grad = requires_grad

    def freeze(x):
        set_requires_grad(x, False)

    def unfreeze(x):
        set_requires_grad(x, True)

    vec = torch.FloatTensor(batch_size, noise_dim).to(device).normal_(0, 1)
    def noise_vector(batch_size):
        return vec

    def frame_noise_vector(batch_size):
        vec = torch.FloatTensor(batch_size, 32, n_frames)\
            .to(device).normal_(0, 1)
        return vec

    def generator_loss(j):
        return 0.5 * ((j - 1) ** 2).mean()

    def train_generator(batch, frames, count):
        zero_grad()

        freeze(d)
        unfreeze(g)

        fake_frames = torch.FloatTensor(1)
        # train frame generator
        # frame_noise = frame_noise_vector(batch_size)
        # fake_frames = g.frames_from_latent(frame_noise)
        # f_j = d.evaluate_latent(fake_frames)
        # loss = generator_loss(f_j)
        # loss.backward()
        # g_frame_optim.step()

        # if count < 10000:
        #     print('gen early return')
        #     return None, None, fake_frames.data.cpu().numpy(), loss.item()

        zero_grad()

        # train time generator
        noise = noise_vector(batch_size)
        # noise = F.pad(batch, (1, 0))[:, :, :-1]
        latent, fake = g(noise)
        d_latent, judgements = d(fake)

        loss = generator_loss(judgements)
        loss.backward()
        g_time_optim.step()
        return \
            latent.data.cpu().numpy(), \
            fake.data.cpu().numpy(), \
            fake_frames.data.cpu().numpy(), \
            loss.item()

    def disc_loss(r_j, f_j):
        return 0.5 * (((r_j - 1) ** 2).mean() + (f_j ** 2).mean())

    def train_discriminator(batch, frames, count):
        zero_grad()
        freeze(g)
        unfreeze(d)

        # train frame disc
        # frame_noise = frame_noise_vector(batch_size)
        # fake = g.frames_from_latent(frame_noise)
        # f_j = d.evaluate_latent(fake)
        # r_j = d.evaluate_latent(batch)
        # loss = disc_loss(r_j, f_j)
        # loss.backward()
        # d_frame_optim.step()

        # if count < 10000:
        #     print('disc early return')
        #     return loss.item(), None, None, None, None, None

        zero_grad()
        # train time generator
        noise = noise_vector(batch_size)
        # noise = F.pad(batch, (1, 0))[:, :, :-1]
        latent, fake = g(noise)
        f_latent, fake_judgements = d(fake)
        r_latent, real_judgements = d(frames)

        loss = disc_loss(real_judgements, fake_judgements)
        loss.backward()
        d_time_optim.step()

        return \
            loss.item(), \
            real_judgements.data.cpu().numpy(), \
            fake_judgements.data.cpu().numpy(), \
            fake.data.cpu().numpy(), \
            f_latent.data.cpu().numpy(), \
            r_latent.data.cpu().numpy()

    def fractal(x, frame_size=16):
        x = x.squeeze()
        x = x.unfold(1, frame_size, frame_size // 2)
        x = x.contiguous().view(-1, frame_size)
        norm = torch.norm(x, dim=-1, keepdim=True)
        x = x / (norm + 1e-12)
        return x

    def fractal2(x, frame_size):
        x = x.squeeze()
        x = x.unfold(1, frame_size, frame_size // 2)
        while x.shape[-1] > 1:
            yield x.contiguous().view(-1, frame_size)
            x = torch.norm(x, dim=-1)
            frame_size = min(x.shape[-1], frame_size)
            step = max(1, frame_size // 2)
            x = x.unfold(1, frame_size, step)

    def cosine_loss(x, y):
        return (-F.cosine_similarity(x, y, dim=-1)).mean()

    def fractal_loss(x, y):
        losses = [F.mse_loss(x, y)]
        frame_size = 4
        for a, b, in zip(fractal2(x, frame_size), fractal2(y, frame_size)):
            losses.append(cosine_loss(a, b))
        return sum(losses)

    def test_dilations(signal_length, max_dilation):
        x = s = torch.ones(1, 1, signal_length)
        kernel = torch.ones(1, 1, 2)
        dilation = 1
        while dilation <= max_dilation:
            x = F.conv1d(F.pad(x, (dilation, 0)), kernel, dilation=dilation)
            print(x.shape)
            dilation *= 2
        return x.data.cpu().numpy().squeeze()

    funcs = cycle([train_generator, train_discriminator])

    real = None
    features = None
    fake_frames = None
    real_frames = None
    fake_features = None
    predicted_frame = None
    real_frame = None
    real_j = None
    fake_j = None
    g_latent = None
    d_f_latent = None
    d_r_latent = None

    # stats_batch = next(ds.batch_stream(512, n_frames))
    # stats_batch = (stats_batch / (np.abs(stats_batch).max(axis=(1, 2), keepdims=True) + 1e-12))
    # spec_mean = stats_batch.mean(axis=(0, 2), keepdims=True)
    # spec_std = stats_batch.std(axis=(0, 2), keepdims=True)

    recon_frames = None
    encoding = None
    orig_frames = None

    def listen_to_recon():
        with torch.no_grad():
            orig = next(ds.batch_stream(1, n_frames))
            orig = \
                (orig / (np.abs(orig).max(axis=(1, 2), keepdims=True) + 1e-12))
            # (1, channels, frames)
            orig = torch.from_numpy(orig)\
                .to(device)\
                .permute(0, 2, 1) \
                .contiguous()\
                .view(-1, 1, 256)

            # mx, _ = torch.max(orig, dim=-1, keepdim=True)
            # orig = orig / (mx + 1e-8)

            encoded, recon = ae(orig)
            # recon = recon * mx
            # (n_frames, 1, 256)
            recon = recon.permute(1, 2, 0).data.cpu().numpy()
            return preview(recon)

    # for count, batch in enumerate(ds.batch_stream(batch_size, n_frames)):
    #     batch = \
    #         (batch / (np.abs(batch).max(axis=(1, 2), keepdims=True) + 1e-12))
    #
    #     batch = torch.from_numpy(batch).to(device)
    #     batch = batch.permute(0, 2, 1).contiguous().view(-1, 1, 256)
    #
    #     # mx, _ = torch.max(batch, dim=-1, keepdim=True)
    #     # batch = batch / (mx + 1e-8)
    #
    #     orig_frames = batch.data.cpu().numpy().squeeze()
    #
    #     ae_optim.zero_grad()
    #     encoded, recon = ae(batch)
    #     encoding = encoded.data.cpu().numpy().squeeze()
    #     recon_frames = recon.data.cpu().numpy().squeeze()
    #     loss = fractal_loss(recon, batch)
    #     loss.backward()
    #     ae_optim.step()
    #     print(loss.item())

    def batch_to_latent(batch):
        batch = \
            (batch / (np.abs(batch).max(axis=(1, 2), keepdims=True) + 1e-12))

        batch_size, channels, frames = batch.shape

        # mx, _ = batch.max(dim=1, keepdim=True)
        # batch = batch / (mx + 1e-8)

        batch = batch.permute(0, 2, 1).contiguous().view(-1, 1, 256)

        with torch.no_grad():
            encoded = ae\
                .encode(batch)\
                .view(batch_size, frames, ae.latent_dim)\
                .permute(0, 2, 1)

        return encoded

    def latent_to_frames(encoded):
        encoded = torch.from_numpy(encoded).to(device)

        batch, latent, frames = encoded.shape
        encoded = encoded\
            .permute(0, 2, 1)\
            .contiguous()\
            .view(batch * frames, latent, 1)

        with torch.no_grad():
            decoded = ae.decode(encoded)
            # (batch * frames, 1, 256)
            decoded = decoded.view(batch, frames, feature_channels).permute(0, 2, 1)

        return decoded.data.cpu().numpy()

    def listen(encoded):
        return preview(latent_to_frames(encoded))

    count = 0
    time_stream = ds.batch_stream(batch_size, n_frames)
    frame_stream = ds.batch_stream(batch_size * n_frames, 1)

    for batch, frames in zip(time_stream, frame_stream):
        frames = frames.reshape(batch_size, n_frames, feature_channels).transpose(0, 2, 1)

        # max one normalization
        frames = (frames / (np.abs(frames).max(axis=(1,), keepdims=True) + 1e-12))
        # max one normalization
        batch = (batch / (np.abs(batch).max(axis=(1,), keepdims=True) + 1e-12))

        # feature-wise normalization
        # batch = batch - spec_mean
        # batch = batch / spec_std

        frames = torch.from_numpy(frames).to(device)
        batch = torch.from_numpy(batch).to(device)

        # batch = batch_to_latent(batch)

        real = batch.data.cpu().numpy()
        real_frames = frames.data.cpu().numpy()

        d_loss, real_j, fake_j, features, d_f_latent, d_r_latent = train_discriminator(batch, frames, count)
        g_latent, fake_features, fake_frames, g_loss = train_generator(batch, frames, count)
        print(f'Batch {count} generator: {g_loss} disc: {d_loss}')
        count += 1


max_length = 512
seed = 3407
batch_size = 32
num_labels = 13
learning_rate = 2e-5
epoch_num = 4

import argparse

parser = argparse.ArgumentParser()
parser.add_argument('-n', type=str)
parser.add_argument('-t', type=str)
parser.add_argument('-i', type=int)
args = parser.parse_args()

if args.n == 0:
    model_name = "bert-base-chinese"
elif args.n == 1:
    model_name = "chinese-bert-wwm"
elif args.n == 2:
    model_name = "chinese-bert-wwm-ext"
elif args.n == 3:
    model_name = "chinese-roberta-wwm-ext"

data_disk_path = "../dataset/preTrain"
dataset_path = data_disk_path + "/dataset_" + str(max_length) + "_head_only"
model_path = data_disk_path + '/' + model_name + '_preTrain_' + args.t + "/checkpoint-" + str(25000 * args.i)
vocab_path = 'hfl/chinese-bert-wwm'
classifier_method = "default"

print(model_path)
import random
import numpy as np
import torch

def seed_everything(seed):
    random.seed(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)
    return seed

seed_everything(seed)

from datasets import load_from_disk

train_dataset = load_from_disk(dataset_path)['train']

from torch.utils.data import DataLoader

train_dataloader = DataLoader(train_dataset, batch_size=batch_size, drop_last=True)

from accelerate import Accelerator

# accelerator = Accelerator()
# accelerator = Accelerator(mixed_precision='fp16')
accelerator = Accelerator(mixed_precision='bf16')
device = accelerator.device

from transformers import BertForSequenceClassification

model = BertForSequenceClassification.from_pretrained(model_path, num_labels=num_labels).to(device)

from transformers import AdamW

def get_parameters(model, model_init_lr, multiplier, classifier_lr):
    parameters = []
    lr = model_init_lr
    for layer in range(12, -1, -1):
        layer_params = {'params': [p for n, p in model.named_parameters() if f'encoder.layer.{layer}.' in n], 'lr': lr}
        parameters.append(layer_params)
        lr *= multiplier
    classifier_params = {
        'params': [p for n, p in model.named_parameters() if 'layer_norm' in n or 'linear' in n or 'pooling' in n],
        'lr': classifier_lr}
    parameters.append(classifier_params)
    return parameters

parameters = get_parameters(model, 2e-5, 0.95, 1e-4)
optimizer = AdamW(parameters, lr=learning_rate)

from transformers import get_linear_schedule_with_warmup

# 学习率调节器
steps_every_epoch = len(train_dataloader)
total_train_steps = steps_every_epoch * epoch_num

scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=0, num_training_steps=total_train_steps)

model, optimizer, training_dataloader, scheduler = accelerator.prepare(model, optimizer, train_dataloader, scheduler)

from tqdm import tqdm

loss_list = []
total_steps = 0

# 训练函数
def train():
    global steps_every_epoch, loss_list, total_steps
    model.train()
    epoch_train_loss = 0
    for batch in tqdm(train_dataloader):
        # 正向传播
        optimizer.zero_grad()

        input_ids = batch['input_ids'].to(device)
        attention_mask = batch['attention_mask'].to(device)
        labels = batch['labels'].to(device)
        outputs = model(input_ids, attention_mask=attention_mask, labels=labels)

        # 低版本transformers库在accelerate库开启混合精度训练后，模型输出的SequenceClassifierOutput不是简单的元组
        # 其中loss与logits封装方式不同，使用以下代码取出需要
        # loss = outputs.loss["loss"]
        loss = outputs[0]
        temp_loss = loss.item()
        epoch_train_loss += loss.item()

        # 反向梯度信息
        accelerator.backward(loss)

        # 参数更新
        optimizer.step()
        scheduler.step()

        total_steps += 1
        if total_steps % 50 == 0:
            loss_list.append(temp_loss)

    print("Epoch: %d, Average training loss: %.4f" % (epoch, epoch_train_loss / steps_every_epoch))

# 验证函数
from matplotlib import pyplot as plt

test_dataset = load_from_disk(dataset_path)['test']
test_dataloader = DataLoader(test_dataset, batch_size=batch_size, drop_last=True)

def validation():
    confusion_matrix = np.zeros((num_labels, num_labels), dtype=int)
    model.eval()
    total_eval_loss = 0
    total_test_steps = len(test_dataloader)
    for batch in tqdm(test_dataloader):
        with torch.no_grad():
            # 正常传播
            input_ids = batch['input_ids'].to(device)
            attention_mask = batch['attention_mask'].to(device)
            labels = batch['labels'].to(device)
            outputs = model(input_ids, attention_mask=attention_mask, labels=labels)

        # 低版本transformers库在accelerate库开启混合精度训练后，模型输出的SequenceClassifierOutput不是简单的元组
        # 其中loss与logits封装方式不同，使用以下代码取出需要
        # loss = outputs.loss["loss"]
        # logits = outputs.loss["logits"]
        loss = outputs[0]
        logits = outputs[1]
        total_eval_loss += loss.item()
        pred_flat = np.argmax(logits.detach().to('cpu').numpy(), axis=1).flatten()
        labels_flat = labels.to('cpu').numpy().flatten()
        assert len(pred_flat) == len(labels_flat)
        for i in range(len(pred_flat)):
            confusion_matrix[labels_flat[i]][pred_flat[i]] = confusion_matrix[labels_flat[i]][pred_flat[i]] + 1

    TP = np.diagonal(confusion_matrix)
    accuracy = np.sum(TP) / np.sum(confusion_matrix, axis=(0, 1))
    FN = np.sum(confusion_matrix, axis=1) - TP
    FP = np.sum(confusion_matrix, axis=0) - TP
    precision = np.nan_to_num(TP / (TP + FP))
    recall = np.nan_to_num(TP / (TP + FN))
    F1_score = np.nan_to_num(2 * precision * recall / (precision + recall))
    macro_precision = np.mean(precision)
    macro_recall = np.mean(recall)
    macro_F1_score = np.mean(F1_score)
    weighted_weight = np.array([35, 402, 392, 57, 71, 1840, 164, 114, 199, 337, 217, 124, 44]) / 3996
    weighted_precision = np.sum(precision * weighted_weight)
    weighted_recall = np.sum(recall * weighted_weight)
    weighted_F1_score = np.sum(F1_score * weighted_weight)

    aver_test_loss = total_eval_loss / total_test_steps
    print("-------------------------------")
    print("Average testing loss: %.4f" % aver_test_loss)
    print("Accuracy: %.4f" % accuracy)
    print("macro Precision: %.4f" % macro_precision)
    print("macro Recall: %.4f" % macro_recall)
    print("macro F1 Score: %.4f" % macro_F1_score)
    print("weighted Precision: %.4f" % weighted_precision)
    print("weighted Recall: %.4f" % weighted_recall)
    print("weighted F1 Score: %.4f" % weighted_F1_score)
    print("-------------------------------")
    # 转化为可以直接粘贴到word的表格的格式
    result = "%.4f\t" % aver_test_loss + "%.4f\t" % accuracy + "%.4f\t" % macro_precision + "%.4f\t" % macro_recall + "%.4f\t" % macro_F1_score + "%.4f\t" % weighted_precision + "%.4f\t" % weighted_recall + "%.4f\t" % weighted_F1_score
    print(result)
    with open("result.txt", "a") as f:
        f.write(result + "\n")
    print("-------------------------------")
    print("confusion_matrix: ")
    for i in range(num_labels):
        for j in range(num_labels):
            if i == j:
                print("%4d*" % confusion_matrix[i][j], end='\t')
            else:
                print("%5d" % confusion_matrix[i][j], end='\t')
        print()
    print("-------------------------------")

    title = model_path.replace("/", "_") + "_" + classifier_method + dataset_path[-14:] + '_train_loss'
    plt.figure(figsize=(20, 8), dpi=80)
    plt.plot(range(len(loss_list)), loss_list)
    plt.xticks(range(len(loss_list)), [i * 50 for i in range(len(loss_list))], rotation=90)
    plt.xlabel("step")
    plt.ylabel("loss")
    plt.title(title)
    plt.tight_layout()
    plt.savefig(title + '.png')
    plt.show()

if __name__ == '__main__':
    for epoch in range(epoch_num):
        print("------------Epoch: %d ----------------" % epoch)
        train()

    validation()


import warnings
import random
from typing import List

import numpy as np
from scipy.stats import beta

from src.operators_utils.Operator import Operator
from src.statistics_utils.Statistics import Statistics
from src.strategies_utils.AdaptiveStrategy import AdaptiveStrategy
from src.strategies_utils.StatisticalStrategy import StatisticalStrategy

class EpsilonGreedy(AdaptiveStrategy):
    def __init__(self, arms: List[Operator], epsilon: float):
        assert 0 <= epsilon <= 1, 'Error: the epsilon parameter for EpsilonGreedy class has to be in [0, 1]'
        super(EpsilonGreedy, self).__init__('eps-greedy-' + str(epsilon), arms)

        self.epsilon = epsilon

    def with_probability_epsilon(self):
        return random.random() < self.epsilon

    def choose_operator_id(self):
        self.probabilities = [self.epsilon for _ in range(self.arms_num)]
        self.probabilities[np.argmax(self.rewards)] = 1 - self.epsilon

        if self.with_probability_epsilon():
            return np.random.randint(0, self.arms_num)
        else:
            return np.argmax(self.rewards)

class UCB1(AdaptiveStrategy):
    def __init__(self, arms: List[Operator]):
        super(UCB1, self).__init__('UCB1', arms)

    def choose_operator_id(self):
        if self.t < self.arms_num:
            return self.t

        arm_id = np.argmax([
            mu_i + np.sqrt(2 * np.log(self.t) / self.pulls[i])
            for i, mu_i in enumerate(self.rewards)
        ])

        self.probabilities = [0 for _ in range(self.arms_num)]
        self.probabilities[arm_id] = 1

        return arm_id

class Softmax(AdaptiveStrategy):
    def __init__(self, arms: List[Operator], tau: float):
        assert tau >= 0, 'Error: the tau parameter for Softmax class has to be in [0, inf]'
        super(Softmax, self).__init__('softmax-' + str(tau), arms)

        self.tau = tau

    def choose_operator_id(self):
        with warnings.catch_warnings():
            warnings.simplefilter("ignore")

            normalization_factor = np.sum([np.exp(mu_j / self.tau) for mu_j in self.rewards])

            probabilities = np.array([
                np.exp(mu_i / self.tau) / normalization_factor
                for mu_i in self.rewards
            ])

            self.probabilities = list(probabilities)

            arm_id = np.random.choice(
                a=[i for i in range(self.arms_num)],
                p=self.probabilities
            )

            return arm_id

class Thompson(AdaptiveStrategy):
    def __init__(self, arms: List[Operator]):
        super(Thompson, self).__init__('thompson', arms)

    def choose_operator_id(self):
        beta_params = zip(self.rewards, self.losses)

        all_draws = [beta.rvs(i[0], i[1], size=1) for i in beta_params]

        self.probabilities = [0 for _ in range(self.arms_num)]
        self.probabilities[all_draws.index(max(all_draws))] = 1

        return all_draws.index(max(all_draws))

class S_n(StatisticalStrategy):
    def __init__(self, operators: List[Operator]):
        super(S_n, self).__init__('s_n', operators, [0.25, 0.25, 0.25, 0.25])

class Random(StatisticalStrategy):
    def __init__(self, operators: List[Operator]):
        super(Random, self).__init__('random', operators, [1 / len(operators) for _ in operators])

class S_fn(StatisticalStrategy):
    def __init__(self, operators: List[Operator]):
        super(S_fn, self).__init__('s_fn', operators, [0.4435, 0.1233, 0.2152, 0.218])

def normalize(raw):
    return [float(i)/sum(raw) for i in raw]

class S_v(StatisticalStrategy):
    def __init__(self, operators: List[Operator], statistics: Statistics):
        super(S_v, self).__init__('s_v', operators,
                                  normalize([statistics.operator_statistics(operator_id)['v'] /
                                             statistics.operator_statistics(operator_id)['t']
                                             for operator_id, _ in enumerate(operators)]))

class S_f(StatisticalStrategy):
    def __init__(self, operators: List[Operator], statistics: Statistics):
        super(S_f, self).__init__('s_f', operators,
                                  normalize([statistics.operator_statistics(operator_id)['f']
                                             for operator_id, _ in enumerate(operators)]))

class S_pool(StatisticalStrategy):
    def __init__(self, operators: List[Operator], statistics: Statistics):
        super(S_pool, self).__init__('s_pool', operators,
                                     normalize([statistics.operator_statistics(operator_id)['is_in_pool']
                                                for operator_id, _ in enumerate(operators)]))

class S_best(StatisticalStrategy):
    def __init__(self, operators: List[Operator], statistics: Statistics):
        super(S_best, self).__init__('s_best', operators,
                                     normalize([statistics.operator_statistics(operator_id)['was_best']
                                                for operator_id, _ in enumerate(operators)]))

class Pure(StatisticalStrategy):
    def __init__(self, operator: Operator):
        super(Pure, self).__init__('pure-' + str(operator.short_name), [operator], [1])


import os
import sys
import shutil
import time
import argparse
import torch
import torch.nn as nn
import torch.nn.parallel
import torch.backends.cudnn as cudnn
import torch.distributed as dist
import torch.optim
import torch.utils.data
import torch.utils.data.distributed
import torchvision.transforms as transforms
import torchvision.datasets as datasets
import torchvision.models as models
import numpy as np
import scripts
sys.path.insert(0, '../')
import random
import scipy.ndimage as nd
import scipy.io as io
import matplotlib
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
from torch.utils import data
from torch.autograd import Variable
import os
import pickle
from collections import OrderedDict
from torch import optim
from torch.autograd import Variable
from torch.utils import data

epochs = 1500
genorator_learning_rate = 0.0025
discriminator_learning_rate = 0.00005
output_size = 32
batch_size = 256

def save_voxels(voxels, path,epoch):
    voxels = voxels[:8].__ge__(0.5)
    fig = plt.figure(figsize=(32, 16))
    gs = gridspec.GridSpec(2, 4)
    gs.update(wspace=0.05, hspace=0.05)

    for i, sample in enumerate(voxels):
        x, y, z = sample.nonzero()
        ax = plt.subplot(gs[i], projection='3d')
        ax.scatter(x, y, z, zdir='z', c='red')
        ax.set_xticklabels([])
        ax.set_yticklabels([])
        ax.set_aspect('equal')
    plt.savefig(path + '/{}.png'.format(str(epoch)), bbox_inches='tight')
    plt.close()

    with open(path + '/{}.pkl'.format(str(epoch)), "wb") as f:
        pickle.dump(voxels, f, protocol=pickle.HIGHEST_PROTOCOL)

def save_model(path, epoch, G, G_optimizer, D, D_optimizer):
    if not os.path.exists(path):
        os.makedirs(path)

    with open(path + "/Generator" + str(epoch) + ".pkl", "wb") as f:
        torch.save(G.state_dict(), f)
    with open(path + "/G_optimizer" + str(epoch) + ".pkl", "wb") as f:
        torch.save(G_optimizer.state_dict(), f)
    with open(path + "/Discriminator" + str(epoch) + ".pkl", "wb") as f:
        torch.save(D.state_dict(), f)
    with open(path + "D_optimizer" + str(epoch) + ".pkl", "wb") as f:
        torch.save(D_optimizer.state_dict(), f)

def z_generator():

    Z = (torch.Tensor(batch_size, 200).normal_(0, 1))
    Z = Z.cuda()

    return Variable(Z)

class Generator(torch.nn.Module):
    def __init__(self):
        super(Generator, self).__init__()
        self.cube_len = 32
        padd = (0, 0, 0)
        bias = False
        if self.cube_len == 32:
            padd = (1,1,1)

        self.layer1 = torch.nn.Sequential(
            torch.nn.ConvTranspose3d(200, self.cube_len*8, kernel_size=4, stride=2, bias=bias, padding=padd),
            torch.nn.BatchNorm3d(self.cube_len*8),
            torch.nn.ReLU()
        )
        self.layer2 = torch.nn.Sequential(
            torch.nn.ConvTranspose3d(self.cube_len*8, self.cube_len*4, kernel_size=4, stride=2, bias=bias, padding=(1, 1, 1)),
            torch.nn.BatchNorm3d(self.cube_len*4),
            torch.nn.ReLU()
        )
        self.layer3 = torch.nn.Sequential(
            torch.nn.ConvTranspose3d(self.cube_len*4, self.cube_len*2, kernel_size=4, stride=2, bias=bias, padding=(1, 1, 1)),
            torch.nn.BatchNorm3d(self.cube_len*2),
            torch.nn.ReLU()
        )
        self.layer4 = torch.nn.Sequential(
            torch.nn.ConvTranspose3d(self.cube_len*2, self.cube_len, kernel_size=4, stride=2, bias=bias, padding=(1, 1, 1)),
            torch.nn.BatchNorm3d(self.cube_len),
            torch.nn.ReLU()
        )
        self.layer5 = torch.nn.Sequential(
            torch.nn.ConvTranspose3d(self.cube_len,1, kernel_size=4, stride=2, bias=bias, padding=(1, 1, 1)),
            torch.nn.Sigmoid()
        )

    def forward(self, x):
        out = x.view(-1, 200, 1, 1, 1)
        out = self.layer1(out)
        out = self.layer2(out)
        out = self.layer3(out)
        out = self.layer4(out)
        out = self.layer5(out)
        return out

class Discriminator(torch.nn.Module):
    def __init__(self):
        super(Discriminator, self).__init__()
        self.cube_len = 32

        padd = (0,0,0)
        bias = False
        if self.cube_len == 32:
            padd = (1,1,1)

        self.layer1 = torch.nn.Sequential(
            torch.nn.Conv3d(1, self.cube_len, kernel_size=4, stride=2, bias=bias, padding=(1, 1, 1)),
            torch.nn.BatchNorm3d(self.cube_len),
            torch.nn.LeakyReLU(0.2)
        )
        self.layer2 = torch.nn.Sequential(
            torch.nn.Conv3d(self.cube_len, self.cube_len*2, kernel_size=4, stride=2, bias=bias, padding=(1, 1, 1)),
            torch.nn.BatchNorm3d(self.cube_len*2),
            torch.nn.LeakyReLU(0.2)
        )
        self.layer3 = torch.nn.Sequential(
            torch.nn.Conv3d(self.cube_len*2, self.cube_len*4, kernel_size=4, stride=2, bias=bias, padding=(1, 1, 1)),
            torch.nn.BatchNorm3d(self.cube_len*4),
            torch.nn.LeakyReLU(0.2)
        )
        self.layer4 = torch.nn.Sequential(
            torch.nn.Conv3d(self.cube_len*4, self.cube_len*8, kernel_size=4, stride=2, bias=bias, padding=(1, 1, 1)),
            torch.nn.BatchNorm3d(self.cube_len*8),
            torch.nn.LeakyReLU(0.2)
        )
        self.layer5 = torch.nn.Sequential(
            torch.nn.Conv3d(self.cube_len*8, 1, kernel_size=4, stride=2, bias=bias, padding=padd),
            torch.nn.Sigmoid()
        )

    def forward(self, x):
        out = x.view(-1, 1, 32, 32, 32)
        out = self.layer1(out)
        out = self.layer2(out)
        out = self.layer3(out)
        out = self.layer4(out)
        out = self.layer5(out)
        return out

class ShapeNetDataset(data.Dataset):

    def __init__(self, root):
        self.root = root
        self.listdir = os.listdir(self.root)

    def __getitem__(self, index):
        with open(self.root + self.listdir[index], "rb") as f:
            volume = np.load(f)
            return torch.FloatTensor(volume)

    def __len__(self):
        return len(self.listdir)

dsets = ShapeNetDataset('data/train/chair/')
dset_loaders = torch.utils.data.DataLoader(dsets, batch_size=batch_size, shuffle=True, num_workers=1)

D = Discriminator()
D.cuda()
G = Generator()
G.cuda()

D_optimizer = optim.Adam(D.parameters(), lr=0.00005, betas=(0.5, 0.5))
G_optimizer = optim.Adam(G.parameters(), lr=0.0025, betas=(0.5, 0.5))
criterion = nn.BCELoss()

pickle_path = './output/pickle/'
loss_result = open("results.txt","w")

for epoch in range(epochs):
        for i, X in enumerate(dset_loaders):

            X = Variable(X.cuda())
            if X.size()[0] != int(batch_size):
                continue
            D.zero_grad()
            G.zero_grad()
            Z = z_generator()

            ground_truth = Variable(torch.ones(batch_size).cuda())
            fake_labels = Variable(torch.zeros(batch_size).cuda())

            d_real = D(X)
            d_real_loss = criterion(d_real, ground_truth)

            fake = G(Z)
            d_fake = D(fake)
            d_fake_loss = criterion(d_fake, fake_labels)

            d_loss = d_real_loss + d_fake_loss

            d_real_acu = torch.ge(d_real.squeeze(), 0.5).float()
            d_fake_acu = torch.le(d_fake.squeeze(), 0.5).float()
            d_total_acu = torch.mean(torch.cat((d_real_acu, d_fake_acu),0))

            if d_total_acu.data[0]<0.95:
                D.zero_grad()
                d_loss.backward()
                D_optimizer.step()

            Z = z_generatorZ()
            fake = G(Z)
            d_fake = D(fake)
            g_loss = criterion(d_fake, real_labels)

            g_loss.backward()
            G_optimizer.step()

        print('Epoch{} , D_loss : {:.4}, G_loss : {:.4}'.format(epoch, d_loss.data[0], g_loss.data[0]))

        loss_result.write('Epoch{} , D_loss : {:.4}, G_loss : {:.4}'.format(epoch,d_loss.data[0], g_loss.data[0]))
        loss_result.write('\n')

        if (epoch + 1) % 10 == 0:

            samples = fake.cpu().data[:8].squeeze().numpy()

            image_path = './output/imag/'
            if not os.path.exists(image_path):
                os.makedirs(image_path)

            save_voxels(samples, image_path,epoch)

        if (epoch + 1) % 100 == 0:
            pickle_save_path = './output/pickle/'
            if not os.path.exists(pickle_save_path):
                os.makedirs(pickle_save_path)
            save_model(pickle_save_path,epoch, G, G_optimizer, D, D_optimizer)


import logging

# Configuración básica de logging
logging.basicConfig(filename='example.log',         # Nombre de archivo para guardar los registros
                    level=logging.DEBUG,            # Nivel de registro mínimo para registrar
                    format='%(asctime)s:%(levelname)s:%(name)s: %(message)s',  # Formato del mensaje
                    filemode='w',                    # Modo de apertura de archivo ('w' sobrescribe, 'a' agrega)
                    datefmt='%Y-%m-%d %H:%M:%S')     # Formato de la fecha en el mensaje

# Registrar mensajes con diferentes niveles de severidad
logging.debug("Este es un mensaje de nivel DEBUG")
logging.info("Este es un mensaje de nivel INFO")
logging.warning("Este es un mensaje de nivel WARNING")
logging.error("Este es un mensaje de nivel ERROR")
logging.critical("Este es un mensaje de nivel CRITICAL")

# Crear un logger personalizado
logger = logging.getLogger("MiLoggerPersonalizado")

# Configurar el nivel de logging para este logger
logger.setLevel(logging.WARNING)

# Crear un manejador de archivo para el logger
file_handler = logging.FileHandler('mi_logger.log')
file_handler.setLevel(logging.WARNING)

# Crear un manejador de consola
console_handler = logging.StreamHandler()
console_handler.setLevel(logging.ERROR)

# Definir un formato para los manejadores
formatter = logging.Formatter('%(name)s - %(levelname)s - %(message)s')
file_handler.setFormatter(formatter)
console_handler.setFormatter(formatter)

# Agregar los manejadores al logger
logger.addHandler(file_handler)
logger.addHandler(console_handler)

# Registrar mensajes con el logger personalizado
logger.debug("Este mensaje no se registrará, el nivel es demasiado bajo")
logger.warning("Este mensaje se registrará en el archivo pero no se mostrará en la consola")
logger.error("Este mensaje se registrará en el archivo y se mostrará en la consola")


"""
Created on 2019/6/18
实现进程间的通信，只有队列q是同一个队列
@author: xieziwei99
"""
import multiprocessing
import os

def sub_task(q, cnt):
	print('子进程号：', os.getpid())
	# cnt = 0
	while cnt < 10:
		q.put('pong')
		cnt += 1

def main():
	print('当前进程号：', os.getpid())
	q = multiprocessing.Queue()
	cnt = 0
	p = multiprocessing.Process(target=sub_task, args=(q, cnt))
	p.start()
	while cnt < 10:
		q.put('ping')
		cnt += 1
	p.join()
	for i in range(20):
		print(q.get(), end=' ')

if __name__ == '__main__':
	main()


import sys
from collections import deque

N, M = map(int, sys.stdin.readline().split())
D = [[0, 1], [1, 0], [0, -1], [-1, 0]]
queue = deque()
a = []
year = 0
check = False

def bfs(start):
    global melt
    queue.append(start)

    while queue:
        r, c = queue.popleft()

        for d in D:
            nr, nc = r + d[0], c + d[1]

            if 0 <= nr < N and 0 <= nc < M:
                if not visited[nr][nc] and a[nr][nc]:
                    queue.append([nr, nc])
                    visited[nr][nc] = 1

                elif a[nr][nc] == 0:
                    melt[r][c] -= 1

for _ in range(N):
    a.append(list(map(int, sys.stdin.readline().split())))

while True:
    visited = [[0] * M for _ in range(N)]
    melt = [[0] * M for _ in range(N)]
    cnt = 0

    for i in range(N):
        for j in range(M):
            if a[i][j] and not visited[i][j]:
                bfs([i, j])
                cnt += 1

    for i in range(N):
        for j in range(M):
            val = a[i][j] + melt[i][j]
            a[i][j] = val if val > 0 else 0

    if cnt >= 2:
        check = True
        break

    if cnt == 0:
        break
    year += 1

if check:
    print(year)
else:
    print(0)


__author__    = "Individual contributors (see AUTHORS file)"
__date__      = "$DATE$"
__rev__       = "$REV$"
__license__   = "AGPL v.3"
__copyright__ = """
This file is part of the ESP Web Site
Copyright (c) 2014 by the individual contributors
  (see AUTHORS file)

The ESP Web Site is free software; you can redistribute it and/or
modify it under the terms of the GNU Affero General Public License
as published by the Free Software Foundation; either version 3
of the License, or (at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public
License along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.

Contact information:
MIT Educational Studies Program
  84 Massachusetts Ave W20-467, Cambridge, MA 02139
  Phone: 617-253-4882
  Email: esp-webmasters@mit.edu
Learning Unlimited, Inc.
  527 Franklin St, Cambridge, MA 02139
  Phone: 617-379-0178
  Email: web-team@learningu.org
"""

from django.core.management import call_command
from django.core.management.base import BaseCommand
from django.conf import settings

import os

class Command(BaseCommand):
    """Update the site.

    - Clean out old .pyc files.
    - Perform database migrations.
    - Install initial data (happens automatically after running migrations).
    - Collect static files.
    - Recompile the theme.
    - Clear memcache
    """
    def handle(self, *args, **options):
        default_options = {
            'verbosity': 1,
            'interactive': False,
            'merge': True,
            'delete_ghosts': True,
            'clear': True,
        }
        default_options.update(options)
        options = default_options

        root = os.path.dirname(os.path.abspath(settings.BASE_DIR))
        file = os.path.join(root, 'esp.wsgi')

        user = os.getenv('USER')
        sudo_user = os.getenv('SUDO_USER')
        # If sudo, we are probably on the live server,
        # so we always want to use www-data
        if sudo_user:
            if user != "www-data":
                raise Exception("Looks like you tried to run this with sudo but without '-u www-data'. Please try again!")
            elif not os.access(file, os.W_OK):
                raise Exception("www-data doesn't have write access for esp.wsgi. Please fix this with chown, then try again!")
        call_command('clean_pyc', **options)
        call_command('migrate', **options)
        call_command('collectstatic', **options)
        call_command('recompile_theme', **options)
        call_command('flushcache', **options)
        os.system("touch " + file)


from gcsa.event import Event
from gcsa.google_calendar import GoogleCalendar
import gcsa

import yaml

config = yaml.load(open("settings.yaml", "r"), Loader=yaml.FullLoader)

calendar_id = config["calendar_id"]

gc = GoogleCalendar(
    calendar_id,
    credentials_path="credentials.json",
    token_path="token.pickle",
)

gc.add_event(
    Event(
        summary="Test event",
        start=gcsa.utils.to_datetime("2021-01-01 00:00:00"),
        end=gcsa.utils.to_datetime("2021-01-01 01:00:00"),
    )
)


#!/usr/bin/env python3

import argparse, os
import numpy as np
import xesmf as xe
from netCDF4 import Dataset
from config_tools import Config

class ConfigRegrid(Config):
    def __init__(self):
        # static config applied for each processing
        self.woa_grid_path = "OSTIA.nc"
        self.woa_lat_var = "lat"
        self.woa_lon_var = "lon"
        self.l4_grid_path = "L4SSS.nc"
        self.l4_lat_var = "lat"
        self.l4_lon_var = "lon"
        self.regridder_file_path = "xesmf_wts.nc"

def parseCommandLine():
    parser = argparse.ArgumentParser(description=("Generate regrid weight file by xesmf"))
    parser.add_argument("config_path", help=("config file path"))
    args = parser.parse_args()
    print(args)
    return args

def main(args):

    cfg = ConfigRegrid()
    cfg.read_cfg(args.config_path, "regrid2")

    # WOA18 latlon grids as output grids
    fn_grd_out = os.path.abspath(cfg.woa_grid_path)
    f = Dataset(fn_grd_out)
    lat1d_grd_out = f.variables[cfg.woa_lat_var][:]
    lon1d_grd_out = f.variables[cfg.woa_lon_var][:]
    f.close()

    print("lat1d_grd_out: shape, min, max=",lat1d_grd_out.shape, np.min(lat1d_grd_out), np.max(lat1d_grd_out))
    print("lon1d_grd_out: shape, min, max=",lon1d_grd_out.shape, np.min(lon1d_grd_out), np.max(lon1d_grd_out))

    # L4 latlon grids as input grids
    fn_grd_in = os.path.abspath(cfg.l4_grid_path)
    f = Dataset(fn_grd_in)
    lat1d_grd_in = f.variables[cfg.l4_lat_var][:]
    lon1d_grd_in = f.variables[cfg.l4_lon_var][:]
    f.close()

    print("lat1d_grd_in: shape, min, max=",lat1d_grd_in.shape, np.min(lat1d_grd_in), np.max(lat1d_grd_in))
    print("lon1d_grd_in: shape, min, max=",lon1d_grd_in.shape, np.min(lon1d_grd_in), np.max(lon1d_grd_in))

    # we only generate wts to WOA18 grids within the L4 grid range; otherwise, xesmf fails to work
    lat_max_allowed = lat1d_grd_out[lat1d_grd_out<lat1d_grd_in.max()][-1]
    idx_lat_max_allowed = np.where(lat1d_grd_out==lat_max_allowed)[0][0]

    lat_min_allowed = lat1d_grd_out[lat1d_grd_out>lat1d_grd_in.min()][0]
    idx_lat_min_allowed = np.where(lat1d_grd_out==lat_min_allowed)[0][0]

    print("min_WOA18_lat can be mapped from L4 ESA: idx_woa18, val_woa18=", idx_lat_min_allowed, lat1d_grd_out[idx_lat_min_allowed])
    print("max_WOA18_lat can be mapped from L4 ESA: idx_woa18, val_woa18=", idx_lat_max_allowed, lat1d_grd_out[idx_lat_max_allowed])

    # generate the regridder
    grd_in = {"lon": lon1d_grd_in, "lat": lat1d_grd_in}
    grd_out = {"lon": lon1d_grd_out, "lat": lat1d_grd_out[idx_lat_min_allowed:idx_lat_max_allowed+1]}
    interp_method = "bilinear"
    periodic = True
    regridder = xe.Regridder(grd_in, grd_out, interp_method, periodic=periodic)

    fn_regridder_out = os.path.abspath(cfg.regridder_file_path)
    print("write regrid info into the file: ", fn_regridder_out)
    regridder.filename = fn_regridder_out
    regridder.to_netcdf()

    print(regridder)

    return None

if __name__ == '__main__':
    args = parseCommandLine()
    main(args)


import sys
import os
sys.path.append(os.path.dirname(os.path.abspath(os.path.dirname(__file__))))
from service.crwaling import lotto_win_history_crawling
from service.compare import data_compare
from service.save import jdbc_config
from datetime import datetime
from dto import util

class WinHistoryImpl:
    def __init__(self, winObject:lotto_win_history_crawling.WinHistoryByArea):
        self.winObject = winObject
        self.jdbcConfig = jdbc_config.JDBCConfig()
        self.utils = util.Utils()

    def compareAndSaveData(self, fristList, secondList):
        lotto645 = data_compare.Lotto645WinInfoCompare()
        others = data_compare.OtherWinInfoCompare()
        l, unknown = lotto645.compareHistoryToStoreInfo(1, fristList["lotto645"])
        l2, unknown2 = lotto645.compareHistoryToStoreInfo(2, secondList["lotto645"])
        self._save_log_win_history(unknown)
        self._save_log_win_history(unknown2)
        ol, unknownOther = others.compareHistoryToStoreInfo(1, fristList)
        ol2, unknownOther2 = others.compareHistoryToStoreInfo(2, secondList)
        self._save_log_win_history(unknownOther)
        self._save_log_win_history(unknownOther2)
        self.utils.write_log_to_file("./log.log", f"l : {len(l)}, {len(unknown)}, l2: {len(l2)}, {len(unknown2)}")
        self.utils.write_log_to_file("./log.log", f"ol : {len(ol)}, {len(unknownOther)}, ol2: {len(ol2)}, {len(unknownOther2)}")
        # self.jdbcConfig.save_win_history_data(l)
        # self.jdbcConfig.save_win_history_data(l2)

        # self.jdbcConfig.save_win_history_data(ol)
        # self.jdbcConfig.save_win_history_data(ol2)

    def getData(self):
        winUtil = lotto_win_history_crawling.WinInfoUtil()
        queryParam = {}
        firstResult = {}
        secondResult = {}
        firstSido = {}
        secondSido = {}
        headers = winUtil.get_win_history_header()
        url = "https://dhlottery.co.kr/store.do"
        session = winUtil.get_session()
        queryParam["method"] = "topStore"

        for key, value in winUtil.lottoTypes.items():
            queryParam["pageGubun"] = value
            firstResult[key] = {}
            secondResult[key] = {}
            for sido in winUtil.address_map.keys():
                firstSigugun, secondSigugun = \
                        self._parseData(session, url, sido, headers, queryParam)
                firstSido[sido] = firstSigugun.copy()
                secondSido[sido] = secondSigugun.copy()
                firstSigugun.clear()
                secondSigugun.clear()
            firstResult[key] = firstSido.copy()
            secondResult[key] = secondSido.copy()
            firstSido.clear()
            secondSido.clear()
        return firstResult, secondResult

    def _parseData(self, session, url, sido, headers, queryParam):
        return self.winObject.parseWinHistory(session, url, sido, headers, queryParam)

    def _save_log_win_history(self, winhistory_data):
        current = datetime.now()
        for item in winhistory_data:
            with open("./error_win_history.txt", "a") as f:
                f.write(f"[{current.strftime('%H:%M:%S')}] ")
                for k,v in item.items():
                    f.write(f"{k}:{str(v)}, ")
                f.write("\n")


# import discord
from discord.ext import commands, tasks

class DbHandler(commands.Cog):
    def __init__(self, bot):
        self.bot = bot
        self.fired_ready = False

    @commands.Cog.listener()
    async def on_ready(self):
        if self.fired_ready:
            return
        self.fired_ready = True
        async for data in self.bot.db.guild_data.find():
            self.bot.guild_data[data["guild_id"]] = data["data"]
        async for data in self.bot.db.channel_data.find():
            self.bot.channel_data[data["channel_id"]] = data["data"]
        self.update_db.start()
        await self.update_db()

    @tasks.loop(minutes=10)
    async def update_db(self):
        for guild, data in self.bot.guild_data.items():
            await self.bot.db.guild_data.update_one(
                {
                    "guild_id": guild,
                },
                {"$set": {"guild_id": guild, "data": data}},
                upsert=True,
            )

        for channel, data in self.bot.channel_data.items():
            await self.bot.db.channel_data.update_one(
                {
                    "channel_id": channel,
                },
                {"$set": {"channel_id": channel, "data": data}},
                upsert=True,
            )

    def cog_unload(self):
        self.update_db.stop()

def setup(bot):
    bot.add_cog(DbHandler(bot))


#!/usr/bin/env python3
# -*- coding:UTF-8

""" 使用管道进行进程间通信

读端-写端,类似与socket端口
每个端口即可以读入又可以写入
"""

import multiprocessing

def consumer(pipe):
    read_p, write_p = pipe
    write_p.close() # 关闭管道的写端
    while True:
        try:
            item = read_p.recv()
        except EOFError:
            break
        print(item)
    print('Consumer done')

def producer(seq,write_p):
    for item in seq:
        write_p.send(item)

if __name__ == '__main__':
    (read_p, write_p) = multiprocessing.Pipe()

    #消费者
    cons_p = multiprocessing.Process(target=consumer, args=((read_p, write_p),))
    cons_p.start()

    #关闭父进程的读端
    read_p.close()

    seq = range(100)
    producer(seq, write_p)

    # 生产完成，关闭生产者的写端
    # 否则，会在消费者的recv处阻塞挂起,important
    write_p.close()

    #等待消费者结束
    cons_p.join()
    print('Main process over')


from urllib.request import urlopen
from bs4 import BeautifulSoup
import ssl

SAMPLE_URL = "http://py4e-data.dr-chuck.net/comments_42.html"
ACTUAL_URL = "http://py4e-data.dr-chuck.net/comments_1511082.html"

# Ignore SSL certificate errors
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE

# url = SAMPLE_URL
# url = ACTUAL_URL
url = input('Enter - ')
html = urlopen(url, context=ctx).read()
soup = BeautifulSoup(html, "html.parser")
total = 0

# Retrieve all of the anchor tags
tags = soup('span')
for tag in tags:
  raw_num = tag.contents[0]
  try:
    num = int(raw_num)
  except:
    continue
  total += num

print(total)


import os
import requests
import datetime

from flask import Flask, flash, jsonify, redirect, render_template, request, session
from flask_socketio import SocketIO, emit, join_room, leave_room

from collections import deque

app = Flask(__name__)
app.config["SECRET_KEY"] = os.getenv("SECRET_KEY")
socketio = SocketIO(app)

CHANNELS = {"Channel": deque([], maxlen=100)}

@app.route("/")
def index():
    """Go to the main page of the webapp"""
    return render_template("index.html")

@socketio.on('connect')
def connection():
    """Shows all active channels"""
    emit('channels', list(CHANNELS.keys()), broadcast=True)

@socketio.on('create channel')
def new_channel(data):
    """Checks if channel exists and adds channel if not existing"""
    selection = data['name']
    if selection in CHANNELS:
        emit('used channel', { "name": selection }, broadcast=True)
    else:
        CHANNELS[selection] = deque(maxlen=100)
        emit('channels', list(CHANNELS.keys()), broadcast=True)

@socketio.on('new message')
def new_message(data):
    """Adds a message to selected channel"""
    time = datetime.datetime.now()
    time = time.replace(microsecond=0)
    if data['channel'] in CHANNELS:
        data['created_at'] = str(time)
        CHANNELS[data['channel']].append(data)
        emit('message', list(CHANNELS[data['channel']]), broadcast=True)

@socketio.on('join')
def on_join(data):
    """User can join room, used for personal touch"""
    username = data['username']
    room = data['channel']
    join_room(room)
    emit('user join', { "username": username }, room=room)

@socketio.on('leave')
def on_leave(data):
    """User can leave room, used for personal touch"""
    username = data['username']
    room = data['room']
    leave_room(room)
    emit('user leave', { "username": username }, room=room)


# -*- coding: utf-8 -*-
import io
import json
import os
import logging
import argparse

import numpy as np
import tensorflow as tf
from tensorflow_datasets.core.features.text import SubwordTextEncoder

from attention import AttentionWeightedAverage
from utils import encode_text_with_encoder, preprocess_text, add_model_argparse, numeric_label_to_text

logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',
                    level=logging.INFO)

def main(FLAGS) -> None:
  logging.info(f"Loading model from {FLAGS.saved_model}")
  model: tf.keras.Model = tf.keras.models.load_model(
    FLAGS.saved_model,
    custom_objects={"AttentionWeightedAverage": AttentionWeightedAverage}
  )
  logging.info(f"Loading encoder from {FLAGS.saved_encoder}")
  encoder_filename: str = FLAGS.saved_encoder.replace(".subwords", '')
  encoder: SubwordTextEncoder = SubwordTextEncoder.load_from_file(encoder_filename)

  sentence: str = preprocess_text(FLAGS.sentence)
  model_input = encode_text_with_encoder(encoder=encoder,
                                         text=sentence,
                                         max_sequence_length=FLAGS.max_words)
  pred: np.ndarray = np.squeeze(model.predict(model_input.reshape(1, -1)))
  logging.debug(f"Predictions: {pred}")

  pred_class: str = numeric_label_to_text(np.round(pred))
  logging.info(f"Input sentence: {FLAGS.sentence}")
  logging.info(f"Classification is '{pred_class}' with a score of {pred}")

  if FLAGS.save_plots:
    logging.info("Visualising classification, saving to 'visualisations' dir")
    os.makedirs("visualisations", exist_ok=True)

    out_model = tf.keras.models.Model(inputs=model.input,
                                      outputs=(model.get_layer("attention").output[1],
                                               model.layers[-1].output))

    attention, posterior = out_model.predict(model_input.reshape(1, -1))

    with io.open(os.path.join("visualisations",
                              "neat_vision_single_string.json"),
                 "w",
                 encoding="utf-8") as m:
      tokens = [encoder.decode([idx]) for idx in encoder.encode(sentence)]
      nest_dict = dict(text=tokens,
                     label=int(np.round(pred)),
                     prediction=int(np.round(pred)),
                     # posterior=posterior[0].tolist(),
                     attention=attention[0].tolist(),
                     id=FLAGS.sentence[:16])
      json.dump([nest_dict], m)

    with io.open(os.path.join("visualisations",
                              "neat_vision_labels.json"),
                 "w",
                 encoding="utf-8") as m:
      json.dump({"0": {"name": "Negative", "desc": "Negative"},
                 "1": {"name": "Positive", "desc": "Positive"}},
                m)
  #   vis_model = tf.keras.models.Model(
  #     inputs=model.input,
  #     outputs=[layer.output for layer in model.layers])
  #
  #   feature_maps = vis_model.predict(model_input)
  #   layer_names_with_index: List[Tuple[int, str]] = [
  #     (index, layer.name)
  #     for index, layer in enumerate(model.layers)
  #     if "dropout" not in layer.name]
  #
  #   for i, layer_name in layer_names_with_index:
  #     save_feature_map(fig=visualise_feature_maps(feature_maps[i],
  #                                                 layer_name),
  #                      output_dir=FLAGS.plot_dir,
  #                      fname=f"{layer_name}.png")
  #
  #   # fig = visualise_feature_maps(feature_maps[1], layer_names_with_index[1][1])
  #   # fig.savefig(fname=os.path.join(FLAGS.plot_dir, "fmap1.png"))
  #
  #   cam_layer_with_index: Tuple[int, str] = [(i, name)
  #                for i, name in layer_names_with_index
  #                if "global_average_pooling" in name or "flatten" in name][-1]
  #   if cam_layer_with_index:
  #     logging.info(f"Class Activation Map Layer: {cam_layer_with_index[1]}")
  #     # We want the input to the layer
  #     cam_layer_with_index = cam_layer_with_index[0] - 1, cam_layer_with_index[1]
  #
  #     cam = get_cam(image_size=model_input.shape[1],
  #                   conv_out=feature_maps[cam_layer_with_index[0]],
  #                   pred_vec=pred,
  #                   all_amp_layer_weights=model.layers[-1].get_weights()[0],
  #                   filters=model.layers[cam_layer_with_index[0]].output.shape[-1])
  #     fig = plot_cam(model_input, cam)
  #     save_feature_map(fig, output_dir=FLAGS.plot_dir, fname=f"cam.png")
  #
  #   FLAGS.test_image.save(os.path.join(FLAGS.plot_dir, "input_image.png"))
  #   greyscale_image.save(os.path.join(FLAGS.plot_dir, "greyscale_input.png"))
  #   rescaled_image.save(os.path.join(FLAGS.plot_dir, "rescaled_model_input.png"))

if __name__ == '__main__':
  parser = argparse.ArgumentParser()
  parser = add_model_argparse(parser)
  parser.add_argument(
    "sentence",
    default="",
    type=str,
    help="String sentence to classify"
  )
  parser.add_argument(
    "--save_plots",
    help="Save visualisations of this classification to "
         "'visualisations' directory",
    action="store_true")
  parser.add_argument(
    "--plot_dir",
    default="visualisations",
    type=str,
    help="Path to save visualisations to (dir)")

  FLAGS, unparsed = parser.parse_known_args()
  if unparsed:
    logging.warning("Unparsed arguments: {}".format(unparsed))

  logging.info("Arguments: {}".format(FLAGS))
  main(FLAGS)


import math
import re
import string
from typing import Iterable, Mapping, Optional, Sequence, Union

import numpy as np
import pandas as pd
import scipy.interpolate

from .. import config, helper

class _FlexMul:
    """Helper class to evaluate "?" in flexible sequences

    In excitation sequences containing multiplication with "?", the "?" is
    replaced by a number so that the excitation sequence length matches
    a given number of frames.
    """
    n_flex_frames: int
    """Number of frames to be filled by repeating sequence part"""

    def __init__(self, n_flex_frames: int):
        """Parameters
        ----------
        n_flex_frames
            Set :py:attr:`n_flex_frames` attribute
        """
        self.n_flex_frames = n_flex_frames

    def __mul__(self, rep_seq: str) -> str:
        """Perform expansion

        `rep_seq` will be repeated until the length of the result is equal to
        :py:attr:`n_flex_frames`.

        Paramters
        ---------
        rep_seq
            Sequence part to be repeated

        Returns
        -------
        Expanded `rep_seq`
        """
        reps, rem = divmod(self.n_flex_frames, len(rep_seq))
        if rem:
            raise ValueError("Number of flexible frames is not divisible by "
                             "length of repeated sequence.")
        return rep_seq * reps

    def __rmul__(self, rep_seq: str) -> str:
        """Same as :py:meth:`__mul__`"""
        return self.__mul__(rep_seq)

class FrameSelector:
    """Select images and datapoints of a certain excitation type

    For instance, if employing alternating excitation for a FRET experiment,
    this can be used to select only the donor or only the acceptor frames.
    For details on how the excitation sequence is specified, see
    :py:attr:`excitation_seq`.

    Examples
    --------
    >>> # Sequence of 8 "images"
    >>> img_seq = numpy.array([numpy.full((3, 3), i) for i in range(8)])
    >>> sel = FrameSelector("odda")
    >>> sel.select(img_seq, "o")
    array([[[0, 0, 0],
            [0, 0, 0],
            [0, 0, 0]],
    <BLANKLINE>
           [[4, 4, 4],
            [4, 4, 4],
            [4, 4, 4]]])
    >>> sel.select(img_seq, "d")
    array([[[1, 1, 1],
            [1, 1, 1],
            [1, 1, 1]],
    <BLANKLINE>
           [[2, 2, 2],
            [2, 2, 2],
            [2, 2, 2]],
    <BLANKLINE>
           [[5, 5, 5],
            [5, 5, 5],
            [5, 5, 5]],
    <BLANKLINE>
           [[6, 6, 6],
            [6, 6, 6],
            [6, 6, 6]]])

    A flexible sequence specification uses ``?`` as a multiplicator. This
    causes repetition of a subsequence such that the excitation sequence
    length matches the image sequence length. In the following example,
    ``"da"`` is repeated three times.

    >>> sel = FrameSelector("o + da * ? + c")
    >>> sel.select(img_seq, "d")
    array([[[1, 1, 1],
            [1, 1, 1],
            [1, 1, 1]],
    <BLANKLINE>
           [[3, 3, 3],
            [3, 3, 3],
            [3, 3, 3]],
    <BLANKLINE>
           [[5, 5, 5],
            [5, 5, 5],
            [5, 5, 5]]])
    """
    excitation_seq: str
    """Excitation sequence. Use different letters to describe different
    excitation types, e.g., "d" for donor excitation, "a" for acceptor,
    etc.

    One needs only specify the shortest sequence that is repeated,
    i. e. ``"ddddaddddadddda"`` is equivalent to ``"dddda"``. It is
    possible to specifiy repetition via the multiplication operator.
    ``"c + da * 3 + c"`` is equivalent to ``"cdadadac"``. Also, flexible
    sequences can be specified using ``?`` as a multiplicator. In ``"c +
    da * ? + c"``, ``"da"`` is repeated an appropriate number of times
    to generate a sequence of given length. This length is either derived
    from the image sequence the class instance is applied to or given
    explicitly (see :py:meth:`select`).

    If the sequence is empty, the FrameSelector does nothing. E.g.,
    :py:meth:`select` and :py:meth:`renumber_frames` just return their
    argument unaltered.
    """

    _rm_whitespace_trans = str.maketrans("", "", string.whitespace)
    """To be used with ``str.translate`` to remove whitespace"""
    _letter_re = re.compile(r"([A-Za-z]+)")
    """Regular expression grouping upper- and lowercase letters"""

    def __init__(self, excitation_seq: str):
        """Parameters
        ----------
        excitation_seq
            Set :py:attr:`excitation_seq`
        """
        self.excitation_seq = excitation_seq

    def _eval_simple(self, seq: str, glob: Mapping = {}, loc: Mapping = {}
                     ) -> str:
        """Evaluate non-flexible sequence

        Parameters
        seq
            Sequence to evaluate
        glob
            Global variables to pass to :py:func:`eval`
        loc
            Local variables to pass to :py:func:`eval`

        Returns
        -------
        Evaluated sequence string
        """
        re_seq = self._letter_re.sub(r'"\1"', seq)
        if not re_seq:
            return ""
        return eval(re_seq, glob, loc)

    def eval_seq(self, n_frames: Optional[int] = None) -> np.ndarray:
        """Evaluate excitation sequence

        Parameters
        ----------
        n_frames
            Number of frames. This needs to be given if sequence is flexible,
            i.e., contains multiplication by ``?``. In this case, ``?``
            evaluates to a number such that the total sequence length is
            `n_frames`. If `n_frames` is equal to -1, ``?`` is replaced by
            1. This is useful for testing and finding e.g. the first occurence
            of a frame without depending on actual data.

        Returns
        -------
        Array of characters representing evaluated sequence
        """
        seq = self.excitation_seq.translate(self._rm_whitespace_trans)
        q_pos = seq.find("?")
        if q_pos < 0:
            eseq = self._eval_simple(seq)
            return np.fromiter(eseq, "U1", len(eseq))
        if n_frames is None:
            raise ValueError("`n_frames` must be given for sequences "
                             "containing '?'")

        pre_pos = seq.rfind("+", 0, q_pos)
        post_pos = seq.find("+", q_pos)

        n_fixed = 0
        if pre_pos >= 0:
            n_fixed += len(self._eval_simple(seq[:pre_pos]))
        if post_pos >= 0:
            n_fixed += len(self._eval_simple(seq[post_pos+1:]))

        if n_frames < 0:
            mul = 1
        else:
            mul = _FlexMul(n_frames - n_fixed)
        eseq = self._eval_simple(seq.replace("?", "_", 1), loc={"_": mul})
        return np.fromiter(eseq, "U1", len(eseq))

    @staticmethod
    def _find_mask(eval_seq: np.ndarray, frame_nos: np.ndarray,
                   which: Union[str, Iterable[str]]) -> np.ndarray:
        """Find frames of certain type in list of frames, return boolean mask

        Parameters
        ----------
        eval_seq
            Evaluated excitation sequence (array of characters).
            See :py:meth:`eval_seq`.
        frame_nos
            Array of frame numbers from which frames should be selected
        which
            Excitation type(s), one or multiple characters out of
            :py:attr:`excitation_seq`.

        Returns
        -------
        Boolean array indicating whether elements of `frame_nos` are of an
        excitation type given by `which`.
        """
        good_frame_mask = np.isin(eval_seq, list(which))
        good_frames = np.nonzero(good_frame_mask)[0]
        mask = np.isin(frame_nos % len(eval_seq), good_frames)
        return mask

    @staticmethod
    def _find_numbers(eval_seq: np.ndarray, frame_nos: np.ndarray,
                      which: Union[str, Iterable[str]]) -> np.ndarray:
        """Find frames of certain type in list of frames, return numbers

        Parameters
        ----------
        eval_seq
            Evaluated excitation sequence (array of characters).
            See :py:meth:`eval_seq`.
        frame_nos
            Array of frame numbers from which frames should be selected
        which
            Excitation type(s), one or multiple characters out of
            :py:attr:`excitation_seq`.

        Returns
        -------
        Elements of `frame_nos` that are of an excitation type given by
        `which`.
        """
        return frame_nos[__class__._find_mask(eval_seq, frame_nos, which)]

    @staticmethod
    def _get_subseq(data: Sequence, index: Sequence[int]) -> Sequence:
        """Get a subsequence of `data`

        Parameters
        ----------
        data
            Sequence to get subsequence from
        index
            Indices of `data` present in subsequence

        Returns
        -------
        If possible, directly index `data` with `index` (works e.g. for numpy
        arrays). If not, create a :py:class:`helper.Slicerator` instance and
        use `index` on that.
        """
        try:
            return data[index]
        except TypeError:
            # This can happen if e.g. data is a list
            s = helper.Slicerator(data)
            return s[index]

    @staticmethod
    def _renumber(eval_seq: np.ndarray, frame_nos: np.ndarray,
                  which: Union[str, Iterable[str]], restore: bool
                  ) -> np.ndarray:
        """Do the actual frame renumbering

        for :py:meth:`select`, :py:meth:`renumber_frames`, and
        :py:meth:`restore_frame_numbers`. If ``restore=False``, any frames that
        don't belong to the excitation type given by `which` are mapped to -1.

        Parameters
        ----------
        eval_seq
            Evaluated excitation sequence (array of characters).
            See :py:meth:`eval_seq`.
        frame_nos
            Array of frame numbers to be mapped
        which
            Excitation type(s), one or multiple characters out of
            :py:attr:`excitation_seq`.
        restore
            If `False`, renumber the frames (as in :py:meth:`select`).
            If `True`, restore original frame numbers (as in
            :py:meth:`restore_frame_numbers`).

        Returns
        -------
        New frame numbers
        """
        if len(eval_seq) == 0 or len(frame_nos) == 0:
            return frame_nos
        frame_nos = np.asanyarray(frame_nos, dtype=int)
        good_frame_mask = np.isin(eval_seq, list(which))
        n_good = np.sum(good_frame_mask)
        max_frame = np.max(frame_nos)
        if restore:
            if not n_good:
                # None of `which` are in excitation seq. There is nothing we
                # can do.
                raise ValueError(f"excitation sequence '{''.join(eval_seq)}' "
                                 f"does not contain types '{''.join(which)}'")
            # f_map_inv needs at least max_frame + 1 entries (the domain of
            # the restoring map is {0, 1, …, max_frame}), meaning
            # good_frame_mask has to be chained (max_frame + 1) / n_good
            # times, where n_good is the number of True entries in
            # good_frame_mask
            n_repeats = math.ceil((max_frame + 1) / n_good)
        else:
            if not n_good:
                # All are invalid and thus mapped to -1
                return np.full(frame_nos.size, -1, dtype=int)
            # f_map needs at least max_frame + 1 entries (the domain of
            # the frame number map is {0, 1, …, max_frame}). f_map_inv's values
            # (which form the codomain of the map) need to go up to at least
            # max_frame.
            n_repeats = math.ceil((max_frame + 1) / len(eval_seq))
        # Calculate the restoring map
        f_map_inv = np.nonzero(np.tile(good_frame_mask, n_repeats))[0]
        if restore:
            return f_map_inv[frame_nos]
        # Invert the restoring map
        f_map = np.full(f_map_inv[-1] + 1, -1, dtype=int)
        f_map[f_map_inv] = np.arange(f_map_inv.size)
        return f_map[frame_nos]

    def renumber_frames(self, frame_nos: np.ndarray,
                        which: Union[str, Iterable[str]],
                        restore: bool = False, n_frames: Optional[int] = None
                        ) -> np.ndarray:
        """Renumber a sequence of frame numbers

        The numbers can be with respect to the original image sequence. In this
        case, set ``restore=False`` and this function will return frame numbers
        with respect to an image sequence which was returned by
        :py:meth:`select`.  Any frames that don't belong to the excitation type
        given by `which` are mapped to -1.
        If ``restore=True`` is set, this works the opposite other way.

        Parameters
        ----------
        frame_nos
            Array of frame numbers to be mapped
        which
            Excitation type(s), one or multiple characters out of
            :py:attr:`excitation_seq`.
        restore
            If `False`, renumber the frames (as in :py:meth:`select`).
            If `True`, restore original frame numbers (as in
            :py:meth:`restore_frame_numbers`).

        Returns
        -------
        New frame numbers
        """
        eval_seq = self.eval_seq(n_frames)
        return self._renumber(eval_seq, frame_nos, which, restore)

    def find_other_frames(self, data: Union[int, Sequence],
                          which: Union[str, Iterable[str]],
                          other: Union[str, Iterable[str]],
                          how: str = "nearest-up") -> Sequence:
        """For given excitation type, find frames of other type

        Typically, this is used to find the closest frame of a certain type,
        e.g., find the next acceptor excitation frame for donor excitation
        frames in FRET ALEX experiments.

        Examples
        --------
        >>> fs = FrameSelector("dddda")
        >>> d_img = fs.select(image_sequence, "d")
        >>> a_img = fs.find_other_frames(image_sequence, "d", "a)
        >>> for di, ai in zip(d_img, a_img):
        ...    # `ai` is the frame of type "a" closest to `di`
        ...    pass

        Parameters
        ----------
        data
            If this is a sequence, return a subsequence of the same length as
            ``select(data, which)``. Each entry of the returned sequence is
            the corresponding entry to what ``select(data, which)`` returns.
            Passing an int is equivalent to using ``numpy.arange(data)``.
        which
            Excitation types for which to find other excitation type's frames
        other
            Other frames' excitation type(s)
        how
            How to do the interpolation. "nearest-up" will return the nearest
            frame. In case of a tie, the “other” frame after the current one
            is used. "nearest" will return the “other” frame before the current
            one in case of a tie. "previous” and "next" return the “other”
            frame before and after the current one, respectively.

        Returns
        -------
        Subsequence of `data` with entries being the “other” entries to
        ``select(data, which)``.
        """
        if isinstance(data, int):
            n_frames = data
            data = None
        else:
            n_frames = len(data)

        eval_seq = self.eval_seq(n_frames)
        idx = np.arange(n_frames)
        source_idx = self._find_numbers(eval_seq, idx, which)
        target_idx = self._find_numbers(eval_seq, idx, other)
        if len(target_idx) == 0:
            raise ValueError("excitation sequence does not contain types any "
                             "of the types in `other`.")
        if len(target_idx) == 1:
            # scipy.interpolate.interp1d needs at least two points, so deal
            # with this case here
            return np.full(len(source_idx), target_idx[0], dtype=int)
        interp_idx = scipy.interpolate.interp1d(
            target_idx, target_idx, how, bounds_error=False,
            fill_value=(target_idx[0], target_idx[-1])
            )(source_idx)
        interp_idx = interp_idx.astype(int)

        if data is not None:
            return self._get_subseq(data, interp_idx)
        return interp_idx

    @config.set_columns
    def select(self, data: Union[pd.DataFrame, Iterable[np.ndarray]],
               which: Union[str, Iterable[str]], renumber: bool = False,
               n_frames: Optional[int] = None, columns: Mapping = {}
               ) -> Union[pd.DataFrame, np.ndarray, helper.Slicerator]:
        """Get only data corresponding to a certain excitation type

        Parameters
        ----------
        data
            Localization data or sequence of images
        which
            Excitation type(s), one or multiple characters out of
            :py:attr:`excitation_seq`.
        renumber
            Renumber frames so that only frames for excitation types
            corresponding to `which` are counted.

            After selecting only data corresponding to an excitation type (e.g.
            "a") in a :py:class:`pandas.DataFrame`, all other types are
            missing and frame number are not consecutive any more. This can be
            a problem for tracking or diffusion analysis. Setting
            ``renumber=True`` will work around this.

            Only applicable if `data` is a :py:class:`pandas.DataFrame`.
        n_frames
            Number of frames. This needs to be given if sequence is flexible,
            i.e., contains multiplication by ``?``. In this case, ``?``
            evaluates to a number such that the total sequence length is
            `n_frames`. If `data` is an image sequence, `n_frames` can be
            inferred from ``len(data)``.

        Returns
        -------
        Selected frames. If `data` is a DataFrame, return only lines
        corresponding to excitation types given by `which`. If
        ``renumber=True``, a modified copy of `data` is returned.

        If `data` is an image sequence, first an attempt is made at
        indexing the images  directly. If `data` supports this (e.g. if it
        is a :py:class:`numpy.ndarray`), the indexed version is returned.
        Otherwise, `img_seq` is converted to
        :py:class:`helper.Slicerator`, indexed, and returned.

        Other parameters
        ----------------
        columns
            Override default column names in case `data` is a
            :py:class:`pandas.DataFrame`. The only relevant name is `time`.
            That means, if the DataFrame has frame number columns "frame2",
            set ``columns={"time": "frame2"}``.
        """
        if not self.excitation_seq:
            return data
        if not isinstance(data, pd.DataFrame) and n_frames is None:
            # Use length of image sequence
            n_frames = len(data)
        eval_seq = self.eval_seq(n_frames)

        if isinstance(data, pd.DataFrame):
            ret = data[
                self._find_mask(
                    eval_seq, data[columns["time"]].to_numpy(), which)]
            if renumber and not ret.empty:
                ret = ret.copy()
                ret[columns["time"]] = self._renumber(
                    eval_seq, ret[columns["time"]].to_numpy(), which, False)
            return ret

        # Deal with image data
        sel_idx = self._find_numbers(eval_seq, np.arange(len(data)), which)
        return self._get_subseq(data, sel_idx)

    @config.set_columns
    def restore_frame_numbers(self, data: pd.DataFrame,
                              which: Union[str, Iterable[str]],
                              n_frames: Optional[int] = None,
                              columns: Mapping = {}):
        """Undo frame renumbering from :py:meth:`select`

        `data` is modified in place for this purpose.

        Parameters
        ----------
        data
            Localization data
        which
            Excitation type(s), one or multiple characters out of
            :py:attr:`excitation_seq`.
        n_frames
            Number of frames. This needs to be given if sequence is flexible,
            i.e., contains multiplication by ``?``. In this case, ``?``
            evaluates to a number such that the total sequence length is
            `n_frames`.

        Other parameters
        ----------------
        columns
            Override default column names in case `data` is a
            :py:class:`pandas.DataFrame`. The only relevant name is `time`.
            That means, if the DataFrame has frame number columns "frame2",
            set ``columns={"time": "frame2"}``.
        """
        if not self.excitation_seq or data.empty:
            return
        data[columns["time"]] = self.renumber_frames(
            data[columns["time"]].to_numpy(), which, True, n_frames)

    def __eq__(self, other: "FrameSelector"):
        return self.excitation_seq == other.excitation_seq


import numpy as np
from scipy import signal
import sys

sys.path.insert(0, '../src')

from core import Sequential, MIMO_Wrapper
from dsp.modem import get_alphabet, Modulator
from dsp.frontend import Upsampler, SRRC_filter
from dsp.optical import CMA
from channels.mimo import Selective_Channel, AWGN
from analysers.scope import IQ_Scope
import matplotlib.pyplot as plt

def generate_H_list(N_tap, N_t, coef=0.3):
    """
    Generate a list of MIMO channel (one matrix by tap)
    """

    I_mat = np.eye(N_t)
    H_list = []
    for indice in range(N_tap):
        B = np.random.randn(N_t, N_t) + 1j*np.random.randn(N_t, N_t)
        H_temp = (coef**indice)*(I_mat + B)
        H_list.append(H_temp)

    return H_list

# parameters
type, M = "PSK", 4
N_t = 2
N_tap = 8
N = 100000
oversampling = 2
rolloff = 0.2
sigma2 = 0.001
nlim = [N-2000, N]

alphabet = get_alphabet(type, M)

# generate siso stream at transmitter side
tx_siso = Sequential([Modulator(alphabet), Upsampler(oversampling), SRRC_filter(rolloff, oversampling)])

# generate random channel
H_list = generate_H_list(N_tap, N_t)

# create chain
channel = Sequential([
            MIMO_Wrapper(tx_siso, N_t),
            Selective_Channel(H_list),
            AWGN(0.001),
            MIMO_Wrapper(IQ_Scope(num="scope1", nlim=nlim), N_t),
            CMA(10, alphabet, mu=0.0001, oversampling=oversampling),
            MIMO_Wrapper(IQ_Scope(num="scope2", nlim=nlim), N_t),
            ])

# simulate communication
S = np.random.randint(0, high=M, size=(2,N))
Y = channel(S)
plt.show()


import pandas as pd
import numpy as np
from math import *
from scipy import spatial
from statistics import stdev
import matplotlib.pyplot as plt
import networkx as nx
import random
#%matplotlib inline
plt.style.use('seaborn-poster')

def filter_list_util(lst1, lst2, common=True):
    filtered_list = []
    for item in lst1:
        if(common==True):
            if(item in lst2):
                filtered_list.append(item)
        if(common==False):
            if (item not in lst2):
                filtered_list.append(item)
    return filtered_list

def construct_task_queue(starting_task, goal_task, job, G):
    tasks_not_placed = []
    topo_path_to_goal = list(nx.topological_sort(G))
    print("Entire topological path to goal: ",topo_path_to_goal)

    #starting_task = input("Enter starting task: ")
    starting_task_prereqs = nx.ancestors(G, starting_task)
    print("prerequisites of the starting task: ", starting_task_prereqs)
    filtered_topo_path_to_goal = filter_list_util(topo_path_to_goal, starting_task_prereqs, common=False)

    #goal_task = input("Enter goal task: ")
    path_to_goal = nx.ancestors(G, goal_task)
    print("path to goal: ",path_to_goal)

    final_filtered_path = filter_list_util(filtered_topo_path_to_goal, path_to_goal)
    print("final filtered path: ",final_filtered_path)

    for task in job:
        if(task not in final_filtered_path):
            tasks_not_placed.append(task)
    return final_filtered_path, tasks_not_placed

#--------------------------------------------------------------------------------------------------------#
#                                   HELPER METHODS                                                       #
#--------------------------------------------------------------------------------------------------------#

def to_Cartesian(lat, lng):
    '''
    function to convert latitude and longitude to 3D cartesian coordinates
    '''
    R = 6371 # radius of the Earth in kilometers

    x = R * cos(lat) * cos(lng)
    y = R * cos(lat) * sin(lng)
    z = R * sin(lat)
    return x, y, z

def deg2rad(degree):
    '''
    function to convert degree to radian
    '''
    rad = degree * 2 * np.pi / 360
    return (rad)

def rad2deg(rad):
    '''
    function to convert radian to degree
    '''
    degree = rad / 2 / np.pi * 360
    return (degree)

def distToKM(x):
    '''
    function to convert cartesian distance to real distance in km
    '''
    R = 6371  # earth radius
    gamma = 2 * np.arcsin(deg2rad(x / (2 * R)))  # compute the angle of the isosceles triangle
    dist = 2 * R * sin(gamma / 2)  # compute the side of the triangle
    return (dist)

def kmToDIST(x):
    '''
    function to convert real distance in km to cartesian distance
    '''
    R = 6371  # earth radius
    gamma = 2 * np.arcsin(x / 2. / R)

    dist = 2 * R * rad2deg(sin(gamma / 2.))
    return (dist)

def Average(lst):
    if (len(lst)==0): return 0
    else:
        cartesian_dist_average = sum(lst) / len(lst)
        km_average_dist = distToKM(cartesian_dist_average)
        return km_average_dist

def standard_deviation(lst):
    if (len(lst) == 0): return 0
    else:
        cartesian_stdev = stdev(lst)
        km_stdev = distToKM(cartesian_stdev)
        return km_stdev

def requestDensity(r, requests):
    PI = 3.142
    if (r==0 or len(requests)==0): return 0
    else:
        area = PI * (r * r)
        density = (len(requests))/area
        return density

def max_dist_from_POIs(POIs_dist_list):
    if (len(POIs_dist_list)==0): return 0
    else:
        cartesian_dist = max(POIs_dist_list)
        km_dist = distToKM(cartesian_dist)
        return km_dist

def remove_outliers(POIs_dist, n):
    if (len(POIs_dist) == 0): return 0
    else:
        for i in range(n):
            cartesian_dist = max(POIs_dist)
            POIs_dist.remove(cartesian_dist)
        final_km_dist = max_dist_from_POIs(POIs_dist)
        return final_km_dist

def main():
    #-------------------------------------------------------------------------------------------------------------#
    #                           READING DATASET AND FILTERING OUT SUSPICIOUS RECORDS                              #
    #-------------------------------------------------------------------------------------------------------------#
    initialData = pd.read_csv('/Users/Tausal21/Desktop/EQ_Works/project/ws-data-spark/data/DataSample.csv')
    num_of_suspicious_records = initialData.duplicated(subset=[' TimeSt', 'Latitude', 'Longitude']).sum()
    print("Number of suspicious records dropped: ", num_of_suspicious_records)
    filteredData = initialData.drop_duplicates(keep=False, subset=[' TimeSt', 'Latitude', 'Longitude']) #filtering out suspicious logins
    print(filteredData.head())

    POIData = pd.read_csv('/Users/Tausal21/Desktop/EQ_Works/project/ws-data-spark/data/POIList.csv')
    print(POIData.head())

    #-------------------------------------------------------------------------------------------------------------#
    #                           CONVERTING LAT/LONG POINTS TO 3D CARTESIAN AND BUILDING KD-TREE                   #
    #-------------------------------------------------------------------------------------------------------------#

    POI_lats = POIData[[' Latitude']].values
    POI_longs = POIData[['Longitude']].values

    POI_x, POI_y, POI_z = zip(*map(to_Cartesian, POI_lats, POI_longs))

    POI_coordinates = list(zip(POI_x, POI_y, POI_z))
    tree = spatial.cKDTree(POI_coordinates)

    POI_01_dist = [] #storing min distances from logins to POI_01
    POI_02_dist = [] #  " " " POI_02
    POI_03_dist = [] #  " " " POI_03
    POI_04_dist = [] #  " " " POI_04
    POI_list = [] #we append this list as a column at the end of the original filtered dataset
    for index, row in filteredData.iterrows():
        x_req, y_req, z_req = to_Cartesian(row['Latitude'], row['Longitude'])
        dist, POI_index = tree.query((x_req, y_req, z_req), 1)

        POI_list.append(POI_index)

        if(POI_index==0): POI_01_dist.append(dist)
        if(POI_index==1): POI_02_dist.append(dist)
        if(POI_index==2): POI_03_dist.append(dist)
        if(POI_index==3): POI_04_dist.append(dist)

    filteredData['Nearest_POI'] = POI_list
    print(filteredData.head())

    #-------------------------------------------------------------------------------------------------------------#
    #                       CALCULATING AVERAGE AND STANDARD DEVIATION FROM EACH POI TO REQ POINT                 #
    #-------------------------------------------------------------------------------------------------------------#
    print("-----------CALCULATING AVERAGE AND STANDARD DEVIATION FROM EACH POI TO REQ POINT---------")

    POI_01_avg_dist_requestPoints = Average(POI_01_dist)
    POI_02_avg_dist_requestPoints = Average(POI_02_dist)
    POI_03_avg_dist_requestPoints = Average(POI_03_dist)
    POI_04_avg_dist_requestPoints = Average(POI_04_dist)
    print("Average real distance in km from POI_01 through 04 \n "
          "to their nearest request points, respectively: ",
          POI_01_avg_dist_requestPoints, POI_02_avg_dist_requestPoints,
          POI_03_avg_dist_requestPoints, POI_04_avg_dist_requestPoints)

    POI_01_stdev_requestPoints = standard_deviation(POI_01_dist)
    POI_02_stdev_requestPoints = standard_deviation(POI_02_dist)
    POI_03_stdev_requestPoints = standard_deviation(POI_03_dist)
    POI_04_stdev_requestPoints = standard_deviation(POI_04_dist)
    print("Standard Deviation in km from POI_01 through 04 \n "
          "to their nearest request points, respectively: ",
          POI_01_stdev_requestPoints, POI_02_stdev_requestPoints,
          POI_03_stdev_requestPoints, POI_04_stdev_requestPoints)

    #-------------------------------------------------------------------------------------------------------------#
    #                       CALCULATING RADIUS AND DENSITY FOR EACH POI CIRCLE                                    #
    #-------------------------------------------------------------------------------------------------------------#
    print("--------CALCULATING RADIUS AND DENSITY FOR EACH POI CIRCLE-------")

    POI_01_circle_rad_km = max_dist_from_POIs(POI_01_dist)
    POI_01_req_density = requestDensity(POI_01_circle_rad_km, POI_01_dist)

    POI_02_circle_rad_km = max_dist_from_POIs(POI_02_dist)
    POI_02_req_density = requestDensity(POI_02_circle_rad_km, POI_02_dist)

    POI_03_circle_rad_km = max_dist_from_POIs(POI_03_dist)
    POI_03_req_density = requestDensity(POI_03_circle_rad_km, POI_03_dist)

    POI_04_circle_rad_km = max_dist_from_POIs(POI_04_dist)
    POI_04_req_density = requestDensity(POI_04_circle_rad_km, POI_04_dist)

    print("Radius of the POIs encompassing all of their nearest req points \n"
          "in real distance(km), respectively: ",
          POI_01_circle_rad_km, POI_02_circle_rad_km,
          POI_03_circle_rad_km, POI_04_circle_rad_km)

    print("Request density of each POI respectively: ",
          POI_01_req_density, POI_02_req_density,
          POI_03_req_density, POI_04_req_density)

    #----------------------------------------------------------------------------------------------------------#
    #                           DRAWING CIRCLES WITH POIs                                                      #
    #----------------------------------------------------------------------------------------------------------#

    POI_01_lat_long = list(POIData.loc[0, [' Latitude', 'Longitude']])
    POI_02_lat_long = list(POIData.loc[1, [' Latitude', 'Longitude']])
    POI_03_lat_long = list(POIData.loc[2, [' Latitude', 'Longitude']])
    POI_04_lat_long = list(POIData.loc[3, [' Latitude', 'Longitude']])

    POI_01_circle = plt.Circle((POI_01_lat_long[1], POI_01_lat_long[0]), POI_01_circle_rad_km, color='blue', alpha=0.1)
    POI_02_circle = plt.Circle((POI_02_lat_long[1], POI_02_lat_long[0]), POI_02_circle_rad_km, color='brown', alpha=0.1)
    POI_03_circle = plt.Circle((POI_03_lat_long[1], POI_03_lat_long[0]), POI_03_circle_rad_km, color='green', alpha=0.1)
    POI_04_circle = plt.Circle((POI_04_lat_long[1], POI_04_lat_long[0]), POI_04_circle_rad_km, color='yellow', alpha=0.1)
    fig, ax = plt.subplots()
    ax.set_xlim((-500, 500))
    ax.set_ylim((-500, 500))
    ax.add_artist(POI_01_circle)
    ax.add_artist(POI_03_circle)
    ax.add_artist(POI_04_circle)
    fig.savefig('POI_circles.png')

    #-------------------------------------------------------------------------------------------------------------#
    #                              REQUEST LOGIN AND POI SCATTERPLOT                                              #
    #-------------------------------------------------------------------------------------------------------------#

    #we extract lat/long for each group of points closest to their respective POIs
    req_lat_long_POI_01 = filteredData.loc[filteredData['Nearest_POI'] == 0]
    req_lat_long_POI_02 = filteredData.loc[filteredData['Nearest_POI'] == 1]
    req_lat_long_POI_03 = filteredData.loc[filteredData['Nearest_POI'] == 2]
    req_lat_long_POI_04 = filteredData.loc[filteredData['Nearest_POI'] == 3]

    p1 = plt.scatter(req_lat_long_POI_01['Longitude'], req_lat_long_POI_01['Latitude'],s=10, c='blue', alpha=0.4)
    POI_01 = plt.scatter(POI_01_lat_long[1], POI_01_lat_long[0], s=800 , c='red', marker='x',edgecolor='black',linewidth=2, alpha=1)

    p2 = plt.scatter(req_lat_long_POI_02['Longitude'], req_lat_long_POI_02['Latitude'],s=10,c='brown', alpha=0.4)
    POI_02 = plt.scatter(POI_02_lat_long[1], POI_02_lat_long[0], s=800 , c='red', marker='x',edgecolor='black',linewidth=2, alpha=1)

    p3 = plt.scatter(req_lat_long_POI_03['Longitude'], req_lat_long_POI_03['Latitude'],s=10,c='green', alpha=0.4)
    POI_03 = plt.scatter(POI_03_lat_long[1], POI_03_lat_long[0], s=800 , c='red', marker='x',edgecolor='black',linewidth=2, alpha=1)

    p4 = plt.scatter(req_lat_long_POI_04['Longitude'], req_lat_long_POI_04['Latitude'],s=10,c='yellow', alpha=0.4)
    POI_04 = plt.scatter(POI_04_lat_long[1], POI_04_lat_long[0], s=800 , c='red', marker='x',edgecolor='black',linewidth=2, alpha=1)

    plt.xlabel('Longitude')
    plt.ylabel('Latitude')
    plt.title('Request login and POI Scatterplot')
    plt.annotate(('POI_01'),xy=(POI_01_lat_long[1], POI_01_lat_long[0]),
                 textcoords='offset points', ha='left', va='top',
                 bbox=dict(boxstyle='round,pad=0.5', fc='brown', alpha=0.4),
                 arrowprops=dict(arrowstyle='->', connectionstyle='arc3,rad=0')
                 )
    '''
    plt.annotate(('POI_02'),xy=(POI_02_lat_long[1], POI_02_lat_long[0]),
                 textcoords='offset points', ha='right', va='bottom',
                 bbox=dict(boxstyle='round,pad=0.5', fc='yellow', alpha=0.5),
                 arrowprops=dict(arrowstyle='->', connectionstyle='arc3,rad=0')
                 )
    '''
    plt.annotate(('POI_03'),xy=(POI_03_lat_long[1], POI_03_lat_long[0]),
                 textcoords='offset points', ha='left', va='top',
                 bbox=dict(boxstyle='round,pad=0.5', fc='brown', alpha=0.4),
                 arrowprops=dict(arrowstyle='->', connectionstyle='arc3,rad=0')
                 )
    plt.annotate(('POI_04'),xy=(POI_04_lat_long[1], POI_04_lat_long[0]),
                 textcoords='offset points', ha='left', va='top',
                 bbox=dict(boxstyle='round,pad=0.5', fc='brown', alpha=0.4),
                 arrowprops=dict(arrowstyle='->', connectionstyle='arc3,rad=0')
                 )
    plt.legend((p1,p2,p3,p4), ('Closest to POI_01', 'Closest to POI_02', 'Closest to POI_03', 'Closest to POI_04'),
               loc='lower left',
               ncol=3,
               fontsize=10)
    plt.show()
    plt.savefig('request_POI_scatterplot.png')

    #----------------------------------------------------------------------------------------------------------#
    #                                         DAG SCHEDULING                                                   #
    #----------------------------------------------------------------------------------------------------------#

    print("------------PIPELINE DEPENDENCY-----------")
    G = nx.DiGraph()

    with open("/Users/Tausal21/PycharmProjects/interview_task3/task_ids.txt", "r") as filestream:
        for line in filestream:
            currentline = line.split(",")
    task_list = currentline

    for node in currentline:
        G.add_node(node)
    print('Graph nodes: ',G.nodes)

    with open("/Users/Tausal21/PycharmProjects/interview_task3/relations.txt", "r") as filestream:
        for line in filestream:
            currentline = line.rstrip('\n').split("->")
            #print(currentline[0], currentline[1])
            G.add_edge(currentline[0], currentline[1])

    #starting and goal tasks are hardcoded 73 and 36 respectively
    m = random.randrange(2,7,1)
    job = random.sample(task_list, m)
    output_task_queue_hardcoded, tasks_not_placed_hardcoded = construct_task_queue('73', '36', job, G)
    print("----this is run with starting and goal tasks as 73 and 36 respectively----")
    print("output task queue: ", output_task_queue_hardcoded)
    print("tasks from the new job not placed: ", tasks_not_placed_hardcoded)

    #user inputs starting and goal task
    starting_task = input("Enter starting task: ")
    goal_task = input("Enter goal task: ")
    output_task_queue, tasks_not_placed = construct_task_queue(starting_task, goal_task, job, G)
    print("-----this is run with user giving starting and goal tasks as input----")
    print("output task queue: ", output_task_queue)
    print("tasks from the new job not placed: ", tasks_not_placed)

if __name__ == '__main__':
    main()


import openpyxl

#1.엑셀파일 만들기
wb = openpyxl.Workbook()

#2.워크시크 만들기(워크시트 이름)
ws = wb.create_sheet('오징어게임')

#3.데이터 추가하기
ws['A1'] = '참가번호'
ws['B1'] = '성명'

ws['A2'] = '001'
ws['B2'] = '오일남'

#4.엑셀 저장하기(엑셀파일 이름)
#wb.save('참가자_data.xlsx') #경로를 따로 지정하지 않으면 현재 프로젝트 폴더에 생성됨
#현재 폴더(프로젝트 폴더 내부의 구분된 폴더)에 저장하고 싶다면 해당폴더의 경로를 앞에 붙여줌
wb.save(r'D:\Crawling_Python\02_파이썬엑셀다루기\참가자_data.xlsx')
#앞에 r을 추가하여 ''안의 문자를 모두 문자열로 취급하라는 뜻

02_파이썬엑셀다루기/01_엑셀만들기.py