import torch
import torch.nn as nn
from torch.nn import functional as F
from torchvision import transforms
from torch.utils.data import DataLoader, Dataset
from PIL import Image
from nltk import word_tokenize
from collections import Counter
import os
from nltk.translate.bleu_score import corpus_bleu
from nltk.translate.meteor_score import single_meteor_score
import pandas as pd
from tqdm import tqdm
import numpy as np

MAX_SEQ_LEN = 100
PRETRAIN_EPOCHS = 30
MEME_EPOCHS = 40
BATCH_SIZE = 10
DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print(f"training on {DEVICE}")
PATIENCE = 10
torch.manual_seed(10)

log_file = 'output/baseline.log'
encoder_path_to_save = 'models/baseline_encoder.pth'
decoder_path_to_save = 'models/baseline_decoder.pth'

# класс для данных пары картинка (путь до картинки) - текст
class Data(Dataset):
    def __init__(self, image_path, data_path, vocab, max_len=MAX_SEQ_LEN, mode='train'):
        self.max_len = max_len
        self.image_path = image_path
        self.image_ids = []
        self.captions = []
        # загружаем id картинок и их captions
        with open(data_path, 'r') as f:
            f.readline()
            for line in f:
                uid, caption = line.strip().split('\t')
                self.captions.append(caption)
                self.image_ids.append(uid)
        self.vocab = vocab
        if mode == 'train':
            self.transform = transforms.Compose([transforms.Resize(256),
                                              transforms.RandomCrop(224),
                                              transforms.RandomHorizontalFlip(),
                                              transforms.ToTensor(),
                                              transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))])
        else:
            self.transform = transforms.Compose([transforms.Resize((224, 224)),
                                                 transforms.ToTensor(),
                                                 transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))])

    def __getitem__(self, index):

        caption = self.captions[index]
        img_id = self.image_ids[index]
        path = os.path.join(self.image_path, f'{img_id}.jpg')
        image = Image.open(path).convert('RGB')
        if self.transform is not None:
            image = self.transform(image)

        tokens = word_tokenize(str(caption).lower())
        caption = []
        # добавляем в описание токен начала
        caption.append(self.vocab('<start>'))
        # добавялем все остальные токены
        caption.extend([self.vocab(token) for token in tokens])
        caption_len = len(caption)
        # если описание слишком длинное - обрезаем его
        if len(caption) > self.max_len - 1:
            caption = caption[:self.max_len - 1]
        # добавляем токен конца
        caption.append(self.vocab('<end>'))
        # если описание недостаточно длинное - добавляем паддинги
        if len(caption) < self.max_len:
            caption.extend([self.vocab('<pad>')] * (self.max_len - len(caption)))

        target = torch.tensor(caption, dtype=torch.long)
        return image, target, caption_len

    def __len__(self):
        return len(self.image_ids)

# класс для хранения словаря слов
class Vocabulary(object):
    def __init__(self):
        self.word2idx = {}
        self.idx2word = {}
        self.idx = 0

    def add_word(self, word):
        if word not in self.word2idx:
            self.word2idx[word] = self.idx
            self.idx2word[self.idx] = word
            self.idx += 1

    def __call__(self, word):
        if word not in self.word2idx:
            return self.word2idx['<unk>']
        return self.word2idx[word]

    def __len__(self):
        return len(self.word2idx)

# функция для построения словаря по имеющимся текстам
def build_vocab(captions, threshold=1):
    counter = Counter()
    for caption in captions:
        tokens = word_tokenize(str(caption).lower())
        counter.update(tokens)

    words = [word for word, cnt in counter.items() if cnt >= threshold]

    vocab = Vocabulary()
    vocab.add_word('<pad>')  # 0
    vocab.add_word('<start>')  # 1
    vocab.add_word('<end>')  # 2
    vocab.add_word('<unk>')  # 3

    for i, word in enumerate(words):
        vocab.add_word(word)
    return vocab

def get_loader(image_path, data_path, vocab, batch_size=BATCH_SIZE, shuffle=True, mode='train'):

    dataset = Data(image_path=image_path, data_path=data_path, vocab=vocab, mode=mode)
    return DataLoader(dataset=dataset, batch_size=batch_size, shuffle=shuffle)

# модель - кодировщик картинки
class EncoderCNN(nn.Module):
    """
    достаем эмбеддинги для картинок с помощью предобученной ResNet
    """

    def __init__(self, embed_size):
        super(EncoderCNN, self).__init__()
        resnet = torch.hub.load('pytorch/vision:v0.10.0', 'resnet50', pretrained=True)
        for param in resnet.parameters():
            param.requires_grad_(False)
        resnet.eval()

        modules = list(resnet.children())[:-1]
        self.resnet = nn.Sequential(*modules)
        self.embed = nn.Linear(resnet.fc.in_features, embed_size)
        self.batch = nn.BatchNorm1d(embed_size, momentum=0.01)

    def forward(self, images):
        features = self.resnet(images)
        features = features.view(features.size(0), -1)
        features = self.batch(self.embed(features))
        return features

# Модель-декодировщик картинки
# лстм, которая принимает на вход эмьеддинг слова и вектор картинки и предсказывает следующее слово
class DecoderRNN(nn.Module):
    def __init__(self, embed_size, hidden_size, vocab_size, num_layers=2):
        """ Initialize the layers of this model."""
        super().__init__()

        self.embed = nn.Embedding(vocab_size, embed_size, padding_idx=0)
        self.lstm = nn.LSTM(input_size=embed_size,
                            hidden_size=hidden_size,  # LSTM hidden units
                            num_layers=num_layers,  # number of LSTM layer
                            bias=True,  # use bias weights b_ih and b_hh
                            batch_first=True,  # input & output will have batch size as 1st dimension
                            dropout=0.2,  # Not applying dropout
                            bidirectional=False,  # unidirectional LSTM
                            )
        self.linear = nn.Linear(hidden_size, vocab_size)
        self.num_layers = num_layers
        self.hidden_size = hidden_size

    def init_hidden(self, batch_size):
        """ At the start of training, we need to initialize a hidden state;
        there will be none because the hidden state is formed based on previously seen data.
        So, this function defines a hidden state with all zeroes
        The axes semantics are (num_layers, batch_size, hidden_dim)
        """
        return (torch.zeros((self.num_layers, batch_size, self.hidden_size), device=DEVICE),
                torch.zeros((self.num_layers, batch_size, self.hidden_size), device=DEVICE))

    def forward(self, features, captions):
        """ Define the feedforward behavior of the model """
        captions = captions[:, :-1]

        # Initialize the hidden state
        batch_size = features.shape[0]  # features is of shape (batch_size, embed_size)
        hidden = self.init_hidden(batch_size)

        # Create embedded word vectors for each word in the captions
        embeddings = self.embed(captions)  # new shape : (batch_size, captions length, embed_size)

        # Stack the features and captions
        embeddings = torch.cat((features.unsqueeze(1), embeddings), dim=1)  # new shape : (batch_size, caption length, embed_size)

        # Get the output and hidden state by passing the lstm over our word embeddings
        # the lstm takes in our embeddings and hidden state
        lstm_out, hidden = self.lstm(embeddings, hidden)  # lstm_out shape : (batch_size, caption length, hidden_size)

        # Fully connected layer
        return self.linear(lstm_out)  # outputs shape : (batch_size, caption length, vocab_size)

    def sample(self, inputs):
        """
        генерация описания под картинку
        каждый раз слово выбирается на основании вероятностей из софтмакса
        """

        output = []
        batch_size = inputs.shape[0]
        hidden = self.init_hidden(batch_size)  # Get initial hidden state of the LSTM

        while len(output) < MAX_SEQ_LEN - 2:
            with torch.no_grad():
                lstm_out, hidden = self.lstm(inputs.unsqueeze(1), hidden)  # lstm_out shape : (1, 1, hidden_size)
                outputs = self.linear(lstm_out)  # outputs shape : (1, 1, vocab_size)
                outputs = outputs.squeeze(1)  # outputs shape : (1, vocab_size)
                p = F.softmax(outputs, dim=1).detach().cpu().numpy()[0]
            max_index = np.random.choice(len(outputs[0]), p=p)  # predict the next word
            output.append(max_index)  # storing the word predicted
            # <end> or <pad> token
            if (max_index == 2 or max_index == 0):
                break

            ## Prepare to embed the last predicted word to be the new input of the lstm
            inputs = self.embed(torch.tensor(max_index).to(DEVICE))  # inputs shape : (1, embed_size)
            inputs = inputs.unsqueeze(0)
        return output

# Function for training
def train():
    # строим словарь
    captions = list(pd.read_csv('data/flickr/train.csv', sep='\t')['text'])
    captions3 = list(pd.read_csv('data/for_training/train.csv', sep='\t')['text'])
    captions.extend(captions3)
    vocab = build_vocab(captions)
    vocab_size = len(vocab)

    # инициализируем кодировщик и декодировщик
    embed_size = 512
    hidden_size = 512
    encoder = EncoderCNN(embed_size)
    decoder = DecoderRNN(embed_size, hidden_size, vocab_size)
    encoder.to(DEVICE)
    decoder.to(DEVICE)

    # инициализируем оптимайзер и лосс-функцию
    criterion = nn.CrossEntropyLoss().to(DEVICE)
    params = list(decoder.parameters()) + list(encoder.embed.parameters()) + list(encoder.batch.parameters())
    optimizer = torch.optim.Adam(params, lr=0.001, betas=(0.9, 0.999), eps=1e-08)

    # сначала в течение PRETRAIN_EPOCHS обучаем сеть на датасете flickr

    train_dataloader = get_loader('data/flickr/images', 'data/flickr/train.csv', vocab)
    val_dataloader = get_loader('data/flickr/images', 'data/flickr/val.csv', vocab, mode='test')

    # функуия, переводящая список или тензор с индексами в предложение
    def covert_idx_to_sent(idx_caption, tensor=True):
        sent = []
        for x in idx_caption:
            if x == 0 or x == 2:
                return sent
            if tensor:
                sent.append(vocab.idx2word[x.item()])
            else:
                sent.append(vocab.idx2word[x])
        return sent

    print("Starting training")

    for epoch in range(PRETRAIN_EPOCHS):

        train_total_loss = 0
        val_total_loss = 0

        # шаг обучения
        for x in tqdm(train_dataloader):
            decoder.train()
            encoder.train()
            images, captions, _ = x
            outputs = decoder(encoder(images.to(DEVICE)), captions.to(DEVICE))
            loss = criterion(outputs.contiguous().view(-1, vocab_size), captions.view(-1).to(DEVICE))
            train_total_loss += loss.item()
            loss.backward()
            nn.utils.clip_grad_norm_(decoder.parameters(), 5)
            nn.utils.clip_grad_norm_(encoder.embed.parameters(), 5)
            nn.utils.clip_grad_norm_(encoder.batch.parameters(), 5)
            optimizer.step()
            optimizer.zero_grad()

        train_total_loss /= len(train_dataloader)

        # шаг оценки модели, лосс на вал датасете  + bleu-3 на нем же
        hypotheses, references = [], []
        for x in tqdm(val_dataloader):
            decoder.eval()
            encoder.eval()
            with torch.no_grad():
                images, captions, lengths = x
                for image, caption, caption_len in zip(images, captions, lengths):
                    ref = decoder.sample(encoder(image.unsqueeze(0).to(DEVICE)))
                    caption = covert_idx_to_sent(caption, tensor=True)
                    references.append([caption])
                    ref = covert_idx_to_sent(ref, tensor=False)
                    hypotheses.append(ref)
                outputs = decoder(encoder(images.to(DEVICE)), captions.to(DEVICE))
                loss = criterion(outputs.contiguous().view(-1, vocab_size), captions.view(-1).to(DEVICE))

                val_total_loss += loss.item()

        # просто печать рандомного реального описания и нашей генерации, чтобы посмотреть какобстоят дела у сети
        print(f"val ref: {references[0]}")
        print(f"val hyp: {hypotheses[0]}")

        val_total_loss /= len(val_dataloader)
        val_bleu = corpus_bleu(references, hypotheses, weights=(0.25, 0.25, 0.25))
        print(f"Epoch: {epoch+1}, train_loss: {train_total_loss}, val_loss: {val_total_loss}, val_bleu-3: {val_bleu}")
        with open(log_file, 'a') as f:
            f.write(f"Epoch: {epoch+1}, train_loss: {train_total_loss}, val_loss: {val_total_loss}, val_bleu-3: {val_bleu}\n")

    ### Далее, дообучаем на описаниях мемов в течение максимум MEME_EPOCHS
    ## patience = PATIENCE, если в течение этого числа эпох bleu-3 не увеличивается, заканчиваем обучение

    with open(log_file, 'a') as f:
        f.write(f"Starting training meme captions\n\n")

    train_dataloader = get_loader('data/images', 'data/for_training/train.csv', vocab)
    val_dataloader = get_loader('data/images', 'data/for_training/val.csv', vocab, mode='test')
    best_bleu, best_epoch = 0, 0

    for epoch in range(MEME_EPOCHS):
        epoch += PRETRAIN_EPOCHS

        train_total_loss = 0
        val_total_loss = 0

        #  шаг обучения
        for x in tqdm(train_dataloader):
            decoder.train()
            encoder.train()
            images, captions, _ = x
            outputs = decoder(encoder(images.to(DEVICE)), captions.to(DEVICE))
            loss = criterion(outputs.contiguous().view(-1, vocab_size), captions.view(-1).to(DEVICE))
            train_total_loss += loss.item()
            loss.backward()
            nn.utils.clip_grad_norm_(decoder.parameters(), 5)
            nn.utils.clip_grad_norm_(encoder.embed.parameters(), 5)
            nn.utils.clip_grad_norm_(encoder.batch.parameters(), 5)
            optimizer.step()
            optimizer.zero_grad()

        train_total_loss /= len(train_dataloader)
        # шаг оценки на валидационном датасете
        hypotheses, references = [], []
        for x in tqdm(val_dataloader):
            decoder.eval()
            encoder.eval()
            with torch.no_grad():
                images, captions, lengths = x
                for image, caption, caption_len in zip(images, captions, lengths):
                    ref = decoder.sample(encoder(image.unsqueeze(0).to(DEVICE)))
                    caption = covert_idx_to_sent(caption, tensor=True)
                    references.append([caption])
                    ref = covert_idx_to_sent(ref, tensor=False)
                    hypotheses.append(ref)
                outputs = decoder(encoder(images.to(DEVICE)), captions.to(DEVICE))
                loss = criterion(outputs.contiguous().view(-1, vocab_size), captions.view(-1).to(DEVICE))

                val_total_loss += loss.item()

        print(f"val ref: {references[0]}")
        print(f"val hyp: {hypotheses[0]}")

        val_total_loss /= len(val_dataloader)
        val_bleu = corpus_bleu(references, hypotheses, weights=(0.25, 0.25, 0.25))
        print(f"Epoch: {epoch+1}, train_loss: {train_total_loss}, val_loss: {val_total_loss}, val_bleu-3(meme): {val_bleu}")

        with open(log_file, 'a') as f:
            f.write(f"Epoch: {epoch+1}, train_loss: {train_total_loss}, val_loss: {val_total_loss}, val_bleu-3(meme): {val_bleu}\n")

        # если у модели bleu-3 лучше, чем у предыдущей лучшей модели - сохраняем
        if best_bleu < val_bleu:
            best_bleu = val_bleu
            best_epoch = epoch
            torch.save(encoder.state_dict(), encoder_path_to_save)
            torch.save(decoder.state_dict(), decoder_path_to_save)
            print(f"saving model at {epoch + 1} epoch ")

        # заканчиваем обучение, если улучшения не было в течение PATIENCE эпох
        if best_epoch + PATIENCE <= epoch:
            print(f"Stopping training at {epoch + 1}")
            break

if __name__ == "__main__":
    train()


from collections import deque

class TreeNode:
    def __init__(self, val) -> None:
        self.val = val
        self.left = None
        self.right = None

# O(n) time | O(n) space
def invertBinaryTree(root):
    queue = deque()
    queue.append(root)
    res = []
    while queue:
        curr = queue.popleft()
        # swap the children nodes before adding them to the queue
        swapLeftandRight(curr)
        if curr.left:
            queue.append(curr.left)
        if curr.right:
            queue.append(curr.right)

        res.append(curr.val)

    return res

def swapLeftandRight(node):
    node.left, node.right = node.right, node.left

## Recursive
# O(n) time  | O(d) depth of recursive stack
def invert_binary_tree(root):
    if root is None:
        return
    swapLeftandRight(root)
    invert_binary_tree(root.left)
    invert_binary_tree(root.right)

root = TreeNode(12)
root.left = TreeNode(7)
root.right = TreeNode(1)
root.left.left = TreeNode(9)
root.right.left = TreeNode(10)
print(invertBinaryTree(root))


#!/usr/bin/env python
# coding: utf-8

# Let's first do a Exploratory Data Analysis and then do a prediction.

# # EDA

# In[ ]:

from IPython.display import clear_output
print()
clear_output()

# In[ ]:

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
sns.set()

# In[ ]:

df = pd.read_csv("../../../input/bobbyscience_league-of-legends-diamond-ranked-games-10-min/high_diamond_ranked_10min.csv")
print(f'Columns: {df.columns}')

# Let's plot the histogram for the gold difference, this shows a correlation between blue side winning and the gold difference beeing positive.

# In[ ]:

sns.displot(data=df, x="blueGoldDiff", hue="blueWins")
print()

# Let's see if there is some correlation into wards placed and the average level.

# In[ ]:

fig, ax = plt.subplots(1,2, figsize=(12,8))

f1 = sns.stripplot(x="redAvgLevel", y="redWardsPlaced", hue="blueWins", data=df, alpha=0.5, ax=ax[0])

f2 = sns.stripplot(x="blueAvgLevel", y="blueWardsPlaced", hue="blueWins", data=df, alpha=0.5, ax=ax[1])

f1.set_xticklabels(f1.get_xticklabels(), rotation=45)
f2.set_xticklabels(f2.get_xticklabels(), rotation=45)
print()

# As expected if a team kills more they end up winning

# In[ ]:

fig, ax = plt.subplots(1,2, figsize=(12,8))

f1 = sns.stripplot(x="redAvgLevel", y="redKills", hue="blueWins", data=df, alpha=0.5, ax=ax[0])

f2 = sns.stripplot(x="blueAvgLevel", y="blueKills", hue="blueWins", data=df, alpha=0.5, ax=ax[1])

f1.set_xticklabels(f1.get_xticklabels(), rotation=45)
f2.set_xticklabels(f2.get_xticklabels(), rotation=45)
print()

# In[ ]:

f1 = sns.stripplot(x="blueAvgLevel", y="blueExperienceDiff", hue="blueWins", data=df, alpha=0.5,)

f1.set_xticklabels(f1.get_xticklabels(), rotation=45)
print()

# Let's see the distribution of heralds and dragons.

# In[ ]:

fig, ax = plt.subplots(2,2, figsize=(14,12))

sns.countplot(data=df, x="redDragons", hue="blueWins", ax=ax[0,0])
sns.countplot(data=df, x="redHeralds", hue="blueWins", ax=ax[0,1])
sns.countplot(data=df, x="blueDragons", hue="blueWins", ax=ax[1,0])
sns.countplot(data=df, x="blueHeralds", hue="blueWins", ax=ax[1,1])
print()

# Probably early on the dragons or heralds doesn't matter that much for winning and the skewed distribution towards no heralds or dragons is mainly because they would be taken after 10 minutes.

# We see an inverse correlation between the Gold diferential for the blue team and the Average level for the red team.

# In[ ]:

ax = sns.catplot(x="redAvgLevel", y="blueGoldDiff", hue="blueWins", data=df, alpha=0.3)
ax.set_xticklabels(rotation=45)
print()

# # Analysing how a team lost with the gold advantage

# We see that there are games that the blue team has an gold advantage and still loses, can we find why searching on other variables?

# In[ ]:

df_bl = df.query('blueWins != 0 & blueGoldDiff > 0')

# In[ ]:

print(f'Red Average Level: {df_bl.redAvgLevel.mean()}')
print(f'Blue Average Level: {df_bl.blueAvgLevel.mean()}')
print(f'Red Average Kills: {df_bl.redKills.mean()}')
print(f'Blue Average Kills: {df_bl.blueKills.mean()}')

# Even though the blue level and gold are higher than the red team, they still lose the game. Let's check the Herald and Dragons for each team:

# In[ ]:

fig, ax = plt.subplots(2,2, figsize=(14,12))

sns.countplot(data=df_bl, x="redDragons", ax=ax[0,0])
sns.countplot(data=df_bl, x="redHeralds", ax=ax[0,1])
sns.countplot(data=df_bl, x="blueDragons", ax=ax[1,0])
sns.countplot(data=df_bl, x="blueHeralds", ax=ax[1,1])
print()

# It seems that I cant' find a exact reason why they loose the game, probably something that occurs after 10 minutes.

# # Useful stats from the data

# In[ ]:

print(f"Probability of winning when you have a gold lead on the blue side: {np.round(len(df.query('blueWins == 1 & blueGoldDiff > 0'))/len(df),3)}")
print(f"Probability of losing when you have a gold lead on the blue side: {np.round(len(df.query('blueWins != 1 & blueGoldDiff > 0'))/len(df),3)}")
print(f"Probability of winning when you have a gold lead on the red side: {np.round(len(df.query('blueWins != 1 & blueGoldDiff < 0'))/len(df),3)}")
print(f"Probability of losing when you have a gold lead on the red side: {np.round(len(df.query('blueWins == 1 & blueGoldDiff < 0'))/len(df),3)}")

# We see that the winning/losing percentage is approximately equal independent of size.

# # Prediction
#
# For the prediction, let's split the features of the blue and red team.

# In[ ]:

red_features = [f for f in df.columns if "red" in f]
blue_features = [f for f in df.columns if "blue" in f]

# Let's predict if the blue team wins or loses based on their features.

# In[ ]:

blue_df = df[blue_features]
blue_df.head()

# Let's split the validation and training data:

# In[ ]:

x = blue_df.drop("blueWins", axis=1)
y = blue_df["blueWins"]

# In[ ]:

from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.33, random_state=42)

# # Random Forest Classifier
#
# Firstly, let's use a random forest classifier for predicting wins for the blue team.

# In[ ]:

from sklearn.ensemble import RandomForestClassifier
randomforest = RandomForestClassifier(n_estimators=100, n_jobs=1, random_state=0)
randomforest.fit(X_train,y_train)

# In[ ]:

from sklearn.metrics import roc_curve, auc

def roc(y_test, y_pred, model_name, title="ROC"):
    """Creates and plots the roc for a model. """

    fpr, tpr, _ = roc_curve(y_test, y_pred)
    roc_auc = auc(fpr, tpr)
    lw = 2
    plt.plot(fpr, tpr, lw=lw, label=f'{model_name} ROC curve area = {roc_auc:0.2f}')
    plt.plot([0, 1], [0, 1], color='red', lw=lw, linestyle='--')
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.title(title)
    plt.legend(loc="lower right")

# For checking if the model works well we will use the ROC curve, if the Area Under the Curve (AUC) is greater than .50, we know that our model works better than a random choice model.

# In[ ]:

from sklearn.metrics import average_precision_score
y_pred_RF = randomforest.predict_proba(X_test)
print(f"Accuracy: {np.around(sum(np.argmax(y_pred_RF, axis=1) == y_test)/len(y_test)*100,1)}%")
average_precision = average_precision_score(y_test, np.argmax(y_pred_RF, axis=1))
roc(y_test, y_pred_RF[:,1], "Random Forest")

# Thus with a simple Random Forest model we can get a robust accuracy of 72% predicting if the blue team wins.


# -*- coding: utf-8 -*-
"""
Created on Wed Apr 12 23:40:48 2023

@author: Saber
"""

import pandas as pd

def Read_Camera(filename):
    i = 1
    fiducial_coordinates = pd.DataFrame(data=[])
    distortions_df = pd.DataFrame(data=[])
    with open(filename, 'r') as f:
        file = f.readlines()

    for line in file:
        if i==1:
         camera_name=line
        elif i == 4:
            postion = line.find(':')
            focal_length = float(line[postion +1:-5])
        elif i < 10:
            o=0
            # do none
        elif i < 18:

            point_name = int(line[ 25:27])
            point_x = float(line[ 28:37])
            point_y = float(line[ 43:52])
            temp_df = pd.DataFrame(data=[{"number": point_name,"point_x": point_x,"point_y": point_y}])
            fiducial_coordinates = pd.concat([fiducial_coordinates, temp_df], axis=0)
        i = i + 1
    distortion_spacing=file[39]
    distortions=file[40]
    i=26
    for ii in range(15):

      if ii<10 :
       ds=int(distortion_spacing[i:i+3])
       d=int((distortions[i:i+3]))
       temp_df = pd.DataFrame(data=[{"distortion_spacing": ds,"distortions": d}])
       distortions_df = pd.concat([distortions_df, temp_df], axis=0)
       i=i+3
      else :
          ds=int(distortion_spacing[i+1:i+4])
          d=int(distortions[i+2:i+4])
          i=i+4
          temp_df = pd.DataFrame(data=[{"distortion_spacing": ds,"distortions": d}])
          distortions_df = pd.concat([distortions_df, temp_df], axis=0)

    return camera_name,focal_length,fiducial_coordinates,distortions_df

import numpy as np
from numpy.linalg import inv
import math as m

def List_Squre(A,L,A_Check,L_Check):
    A=np.array(A)
    L=np.array(L)
    AT=A.transpose()
    X=(inv(AT.dot(A)).dot(AT)).dot(L)
    V_Control=A.dot(X)-L
    Rmse_Control=m.sqrt((V_Control.transpose().dot(V_Control))/len(V_Control))
    try:
        V_Check=np.array(A_Check).dot(X)-L_Check
        Rmse_Check=m.sqrt((V_Check.transpose().dot(V_Check))/len(V_Check))
    except:
        V_Check=''
        Rmse_Check=''
    return X,V_Control,Rmse_Control,V_Check,Rmse_Check

#---------------------------------------------------------------------------------------------------------

from pandasql import sqldf
mysql = lambda q: sqldf(q, globals())

import cv2

class Interior_Orientation:

    def __init__(self,ImagePath):
          self.img = cv2.imread(ImagePath)
          self.Gry =(cv2.cvtColor(self.img, cv2.COLOR_BGR2GRAY))
          self.Laplacian  = cv2.Laplacian(self.Gry ,cv2.CV_64F)
          self.ptsLeft =[]
          self.ptsRight=[]

#------------------------------------------------------------------------------------------------------------------
    def organize_point(self ,xy_ob_input,control_list,Camera_Path,investiy=False):
        self.camera_name,self.focal_length,self.fiducial_coordinates,self.Camera_distortions=Read_Camera(Camera_Path)

        global xy_obs
        global xy_true
        global Control_list
        global Check_list
        global AllPoint
        xy_obs,xy_true=xy_ob_input,self.fiducial_coordinates
        Control_list=pd.DataFrame(control_list,columns=['Chose_number'])
        Check_list=pd.DataFrame(control_list,columns=['Chose_number'])

        query='''
            SELECT xy_obs.number number,  xy_obs.point_x x_obs,  xy_obs.point_y  y_obs, xy_true.point_x  x_true, xy_true.point_y  y_true
            from  xy_obs , xy_true
            where xy_true.number=xy_obs.number;
        '''
        AllPoint=mysql(query)
        query='''
        SELECT *
        from  AllPoint
        where AllPoint.number IN (select * from Control_list);
        '''
        self.Control_Points=mysql(query)
        query='''
        SELECT *
        from  AllPoint
        where AllPoint.number NOT IN (select * from Control_list);
        '''
        self.Check_Points=mysql(query)

# ------------------------------------------------------------------------------------------------------------------------
    def Mapping_Method(self,Method):
        L,A,L_Check,A_Check=[],[],[],[]
        self.Method=Method
        R=0
        for xp,yp,XC,YC in zip(self.Control_Points.x_obs,self.Control_Points.y_obs,self.Control_Points.x_true,self.Control_Points.y_true):
            L.append(XC)
            L.append(YC)

            if Method=='Comformal':
                A.append([xp, yp,1,0])
                A.append([yp,-xp,0,1])

            if Method=='Afine':
                A.append([xp,yp,1,0,0,0])
                A.append([0,0,0,xp,yp,1])

            if Method=='Projetive':
                A.append([xp,yp,1,0,0,0,-xp*XC,-yp*XC])
                A.append([0,0,0,xp,yp,1,-xp*YC,-yp*YC])

        for xp,yp,XC,YC in zip(self.Check_Points.x_obs,self.Check_Points.y_obs,self.Check_Points.x_true,self.Check_Points.y_true):
            L_Check.append(XC)
            L_Check.append(YC)

            if Method=='Comformal':
                A_Check.append([xp,yp,1,0])
                A_Check.append([yp,-xp,0,1])

            if Method=='Afine':
                A_Check.append([xp,yp,1,0,0,0])
                A_Check.append([0,0,0,xp,yp,1])

            if Method=='Projetive':
                A_Check.append([xp,yp,1,0,0,0,-xp*XC,-yp*XC])
                A_Check.append([0,0,0,xp,yp,1,-xp*YC,-yp*YC])

        self.X,self.V_Control,self.Rmse_Control,self.V_check,self.Rmse_Check=List_Squre(A,L,A_Check,L_Check)

    def Mapping_Method_to_picel(self):
       L,A,L_Check,A_Check=[],[],[],[]
       Method=self.Method
       R=0
       for xp,yp,XC,YC in zip(self.Control_Points.x_true,self.Control_Points.y_true,self.Control_Points.x_obs,self.Control_Points.y_obs):
           L.append(XC)
           L.append(YC)

           if Method=='Comformal':
               A.append([xp, yp,1,0])
               A.append([yp,-xp,0,1])

           if Method=='Afine':
               A.append([xp,yp,1,0,0,0])
               A.append([0,0,0,xp,yp,1])

           if Method=='Projetive':
               A.append([xp,yp,1,0,0,0  ,-xp*XC,-yp*XC])
               A.append([0,0,0  ,xp,yp,1,-xp*YC,-yp*YC])

       for xp,yp,XC,YC in zip(self.Check_Points.x_true,self.Check_Points.y_true,self.Check_Points.x_obs,self.Check_Points.y_obs):
           L_Check.append(XC)
           L_Check.append(YC)

           if Method=='Comformal':
               A_Check.append([xp,yp,1,0])
               A_Check.append([yp,-xp,0,1])

           if Method=='Afine':
               A_Check.append([xp,yp,1,0,0,0])
               A_Check.append([0,0,0,xp,yp,1])

           if Method=='Projetive':
               A_Check.append([xp,yp,1,0,0,0,-xp*XC,-yp*XC])
               A_Check.append([0,0,0,xp,yp,1,-xp*YC,-yp*YC])
       self.X_to_pixcel,self.V_Control_to_pixcel,self.Rmse_Control_to_pixcel,self.V_check_to_pixcel,self.Rmse_Check_to_pixcel=List_Squre(A,L,A_Check,L_Check)
#-------------------------------------------------------------------------------------------------------------------------
    def Mapping_to_point(self,xp,yp):
        Method=self.Method
        x_o,y_o=0,0
        if Method=='Comformal':
            x_o=(self.X[0]*xp +self.X[1]*yp +self.X[2])
            y_o=(-self.X[1]*xp+self.X[0]*yp+self.X[3])
        if Method=='Afine':
            x_o=(self.X[0]*xp+self.X[1]*yp+self.X[2])
            y_o=(self.X[3]*xp+self.X[4]*yp+self.X[5])
        if Method=='Projetive':
            x_o=(self.X[0]*xp+self.X[1]*yp+self.X[2])/(self.X[6]*xp+self.X[7]*yp +1)
            y_o=(self.X[3]*xp+self.X[4]*yp+self.X[5])/(self.X[6]*xp+self.X[7]*yp +1)
        xy={'x':x_o,"y":y_o}
        try:xy=pd.DataFrame(xy)
        except:xy=xy
        return xy

    def Mapping_to_Pixcel(self,xp ,yp):
       # xp: np.array or number
       # yp: np.array or number
        Method=self.Method
        x_o,y_o=0,0
        if Method=='Comformal':
            x_o=(self.X_to_pixcel[0]*xp-self.X_to_pixcel[1]*yp +self.X_to_pixcel[2])
            y_o=(self.X_to_pixcel[1]*xp+self.X_to_pixcel[0]*yp+self.X_to_pixcel[3])
        if Method=='Afine':
            x_o=(self.X_to_pixcel[0]*xp+self.X_to_pixcel[1]*yp+self.X_to_pixcel[2])
            y_o=(self.X_to_pixcel[3]*xp+self.X_to_pixcel[4]*yp+self.X_to_pixcel[5])
        if Method=='Projetive':
            x_o=(self.X_to_pixcel[0]*xp+self.X_to_pixcel[1]*yp+self.X_to_pixcel[2])/(self.X_to_pixcel[6]*xp+self.X_to_pixcel[7]*yp+1)
            y_o=(self.X_to_pixcel[3]*xp+self.X_to_pixcel[4]*yp+self.X_to_pixcel[5])/(self.X_to_pixcel[6]*xp+self.X_to_pixcel[7]*yp+1)
        xy={'x':x_o,"y":y_o}
        try:xy=pd.DataFrame(xy)
        except:xy=xy
        return xy

#-----------------------------------------------------------------------------------------------------------------------------------------
    def Fiducial_Mark(self):
        matchCase=[]
        def click_event(event, x, y, flags, params):
             if event == cv2.EVENT_LBUTTONDOWN: # checking  left mouse clicks
                 global point
                 point=[x,y]
                 matchCase.append(point)
                 # displaying the point on the image window
                 cv2.circle(self.img, (x,y), 1, (0,0,255), -1)
                 cv2.imshow(win_name,self.img)
             if event == cv2.EVENT_RBUTTONDOWN: # checking  left mouse clicks

               #  matchCase.remove(0)
                 # displaying the point on the image window
                 cv2.circle(self.img, (x,y), 1, (0,0,255), -1)
                # cv2.imshow(win_name,self.img)
                 cv2.destroyAllWindows()

        def show():

            cv2.namedWindow(win_name, cv2.WINDOW_NORMAL)

            # Move it to (X,Y)
            cv2.moveWindow(win_name, 0, 0)

            # Show the Image in the Window
            cv2.imshow(win_name, self.img)

            # Resize the Window
            cv2.resizeWindow(win_name, 400, 300)

        #    cv2.imshow(window_title, self.img)

            cv2.setMouseCallback(win_name, click_event)

            cv2.waitKey(0)
            # close the window
            cv2.destroyAllWindows()
        win_name='chose 8 maker'
        show()
        return matchCase

        #self.img_rotated = cv2.warpAffine(self.Gry, R, (cols,rows))

Image_Path = {'left':r'D:\Projects\2.Term2\Digital_Photogrammetry\project1\main\data\21_60_.png',
              'right':r'D:\Projects\2.Term2\Digital_Photogrammetry\project1\main\data\22_60_.png'}
Fiducial_Mark_Path = {'left':r'D:\Projects\2.Term2\Digital_Photogrammetry\project1\main\data\Fiducial_Mark_On_Left_Image_readed.txt',
                      'right':r'D:\Projects\2.Term2\Digital_Photogrammetry\project1\main\data\Fiducial_Mark_On_Right_Image_readed.txt'}

Camera_Path=r"D:\Projects\2.Term2\Digital_Photogrammetry\project1\main\data\CAMERA.txt"

# Sensors_R=Interior_Orientation(Image_Path['right'],Camera_Path)

# cc=Sensors_R.Fiducial_Mark()


import requests
import time

# api-endpoint
API_ENDPOINT = "https://challenge.crossmint.io/api/polyanets"

# candidateId
CANDIDATE_ID = "95ccae45-102c-4748-98f2-b06b2b3aa93e"

# function to make_cross
# (it also makes delete request in case there were prior polyanets in canvas)
# n defines the cross length, which is the length from center
# n = 2 would be:
# x x
#  x
# x x
def make_cross(n):
    # since it's a 11x11 canvas, max n should be 6
    if  n < 1 or n > 6 :
        print("Error: n should be between 1 and 6")
        return
    for i in range(11):
        for j in range(11):

            data = {
                "candidateId": CANDIDATE_ID,
                "row": i,
                "column": j
            }

            print(f"modyfing row: {i} column: {j}")

            # verifies it is on the cross of a 11x11 grid and makes the space acording to n
            if (i > (5 - n)) and (i < (5 + n)) and (j == i or j == (10 - i)):
                r = requests.post(url = API_ENDPOINT, data = data)
            else:
                r = requests.delete(url = API_ENDPOINT, data = data)

            print("Response: ")
            print(r.json())

            # wait one sec bewteen requests to avoid "Too many requests"
            print("waiting 1 second...")
            time.sleep(1)

make_cross(4)


#!/usr/bin/env python

import logging
import kubernetes
import os
import random
import time
import pytz
import json
import datetime

from kubernetes.client.rest import ApiException
from http import HTTPStatus

KILL_FREQUENCY = int(os.environ.get('CHAOS_MONKEY_KILL_FREQUENCY_SECONDS', 300))
LOGGER = logging.getLogger(__name__)

logging.basicConfig(level=logging.INFO)

# No error handling, if things go wrong Kubernetes will restart for us!

kubernetes.config.load_incluster_config()
v1 = kubernetes.client.CoreV1Api()

while True:
    pods = v1.list_pod_for_all_namespaces().items
    pod = random.choice(pods)
    LOGGER.info("Terminating pod %s/%s", pod.metadata.namespace, pod.metadata.name)
    event_name = "Chaos monkey kill pod %s" % pod.metadata.name
    v1.delete_namespaced_pod(
        name=pod.metadata.name,
        namespace=pod.metadata.namespace,
        body=kubernetes.client.V1DeleteOptions(),
    )
    event_timestamp = datetime.datetime.now(pytz.utc)
    try:
        event = v1.read_namespaced_event(event_name, namespace=pod.metadata.namespace)
        event.count += 1
        event.last_timestamp = event_timestamp
        v1.replace_namespaced_event(event_name, pod.metadata.namespace, event)
    except ApiException as e:
        error_data = json.loads(e.body)
        error_code = HTTPStatus(int(error_data['code']))
        if error_code == HTTPStatus.NOT_FOUND:
            new_event = kubernetes.client.V1Event(
                count=1,
                first_timestamp=event_timestamp,
                involved_object=kubernetes.client.V1ObjectReference(
                    kind="Pod",
                    name=pod.metadata.name,
                    namespace=pod.metadata.namespace,
                    uid=pod.metadata.uid,
                ),
                last_timestamp=event_timestamp,
                message="Pod deleted by chaos monkey",
                metadata=kubernetes.client.V1ObjectMeta(
                    name=event_name,
                ),
                reason="ChaosMonkeyDelete",
                source=kubernetes.client.V1EventSource(
                    component="chaos-monkey",
                ),
                type="Warning",
            )
            v1.create_namespaced_event(namespace=pod.metadata.namespace, body=new_event)
        else:
            raise
    time.sleep(KILL_FREQUENCY)


import requests
from bs4 import BeautifulSoup

pageNum = 1

base_url = 'https://hiculturearts.pastperfectonline.com'

url = '{}/search?page={}&utf8=%E2%9C%93&search_criteria=*&searchButton=Search'.format(base_url, pageNum)

r = requests.get(url)

while 'No results found' not in r.text:
    soup = BeautifulSoup(r.text, 'html.parser')

    results = soup.findAll("div", {"class": "indvImage"})
    for result in results:
        for a in result.find_all('a', href=True):
            print(base_url + a['href'])
    pageNum += 1
    url = '{}/search?page={}&utf8=%E2%9C%93&search_criteria=*&searchButton=Search'.format(base_url, pageNum)
    r = requests.get(url)


import sys
import importlib

class LasyModule:
    def __init__(self, modulename):
        self.modulename = modulename
        self.module = None

    def __getattr__(self, attr):
        if self.module is None:
            try:
                self.module = importlib.import_module(self.modulename)
            except ImportError:
                print(f"LazyModule: {self.modulename} is missing.", file=sys.stderr)
                raise

        return self.module.__getattribute__(attr)

xarray = LasyModule('xarray')
diskcache = LasyModule('diskcache')


# -*- coding: utf-8 -*-
"""
Created on Sun Mar 21 21:55:17 2021

@author: MengZY
"""

import numpy as np

measure = np.load('X_final.npy')
label = np.load('label.npy')

from sklearn.model_selection import KFold
kf = KFold(n_splits = 5, shuffle = True)

from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import roc_auc_score, classification_report
import time

for train_index, test_index in kf.split(measure):

    X_train, X_test = measure[train_index], measure[test_index]
    y_train, y_test = label[train_index], label[test_index]
    start = time.time()
    clf = MultinomialNB().fit(X_train, y_train)
    end = time.time()
    print('GauNB time: ', end-start)
    print('GauNB report: ', classification_report(y_test,clf.predict(X_test)))
    print('GauNB auc: ', roc_auc_score(y_test, clf.predict_proba(X_test)[:,1]))


import numpy as np
import asyncio
import datetime as dt

from ..utils.spear3_api import within_spear3, _get_lifetime
from ..utils.spear3_lt_opt import _set_origin, x_to_pv

async def _get_objective(x, connection):
    assert len(x) == 13, 'Input dimension must be equal to 13'

    # set quadrupole PV values
    df = _set_origin(x.reshape(1, -1), connection)
    await asyncio.sleep(2)

    # calculate objective
    lifetime = await _get_lifetime(connection)
    obj = np.array([-lifetime])

    return obj

@within_spear3
async def get_objective(x, connection):
    return await _get_objective(x, connection)

def evaluator_generator(connection):
    async def evaluate(X, hook):
        assert type(X) == np.ndarray, 'Input X must be a numpy array'

        PV = x_to_pv(X)

        Y = np.zeros((PV.shape[0], 1))
        for i, pv in enumerate(PV):
            obj = await get_objective(pv, connection)
            finished_time = dt.datetime.now().strftime("%m-%d-%Y_%H-%M-%S")
            Y[i] = obj
            if hook:
                hook(pv, obj, finished_time)

        return Y

    return evaluate


import os
import time
import random
import json

import youtube_dl

def channel(key):
    channel_list = []
    with open('{}.json'.format(key), 'r') as f:
        data = json.load(f)
        channel_list.append(data)
    return channel_list

def download(url, folder, title):

    video_info = youtube_dl.YoutubeDL().extract_info(url, download=False)
    formats = video_info.get('formats')
    format_list = []
    for i in formats:
        a = i.get('format').split(' - ')[0]
        format_list.append(a)

    ydl_opts = {
        'outtmpl': '/root/{}/{}'.format(folder, title)
    }

    # 检测480p分辨率，如果没有就用默认的。
    if '135' in format_list:
        ydl_opts['format'] = '135+m4a'
    else:
        ydl_opts['format'] = 'mp4+m4a'

    with youtube_dl.YoutubeDL(ydl_opts) as ydl:
        result = ydl.extract_info(url, download=True)

if __name__ == '__main__':

    a = 'vlog 动画 日本医疗'

    for key in a.split(' '):
        channel_list = channel(key)

        for i in channel_list:

            keyword = i['keyword']
            down_nums = i['down_nums']
            details_list = i['details']

            # 创建目录
            file_path = os.path.exists('/root/{}'.format(keyword))
            if not file_path:
                os.mkdir('/root/{}'.format(keyword))
                print('{}'.format(keyword)+'目录创建成功')

            else:
                print('{}'.format(keyword)+'目录已存在')

            # 下载失败记录
            failure_list = []

            # 开始下载
            for video in details_list:
                title_set = video['title_set']
                title_set_mp4 = title_set + '.mp4'
                href = video['href']

                mp4_list = []
                for root,dirs,files in os.walk('/root/{}'.format(keyword)):

                    for mp4 in files:
                        if '.mp4' in mp4:
                            mp4_list.append(mp4)

                # 已经下过的视屏，包含失败的
                down_video_list = mp4_list + failure_list

                if len(mp4_list)<down_nums:
                    if title_set_mp4 not in down_video_list:
                        try:
                            download(href, keyword, title_set)
                            print(details_list.index(video)+1)
                        except:
                            # 保存表
                            failure_list.append(title_set_mp4)
                            print('{}下载失败'.format(details_list.index(video)+1))

                            time.sleep(random.randint(1, 3))  # 限制爬取时间


#!/usr/bin/env python
# coding: utf-8

# In[2]:

import numpy as np
import matplotlib.pyplot as plt
import array as arr
import string
import os
import pickle
from wordcloud import WordCloud
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn import svm
from nltk import PorterStemmer
from nltk.stem import WordNetLemmatizer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import confusion_matrix
stemmer = PorterStemmer()
lemmatizer = WordNetLemmatizer()

stop_words = ["i", "me", "my", "myself", "we", "our", "ours", "ourselves", "you", "your", "yours", "yourself",
              "yourselves", "he", "him", "his", "himself", "she", "her", "hers", "herself", "it", "its", "itself",
              "they", "them", "their", "theirs", "themselves", "what", "which", "who", "whom", "this", "that", "these",
              "those", "am", "is", "are", "was", "were", "be", "been", "being", "have", "has", "had", "having", "do",
              "does", "did", "doing", "a", "an", "the", "but", "and", "if", "or", "because", "as", "until", "while",
              "of", "at", "by", "for", "with", "about","against", "between", "into", "through", "during", "before",
              "after", "above", "below", "to", "from", "up", "down", "in", "out", "on", "off", "over", "under",
              "further", "then", "once", "here", "there", "when", "where", "all", "both", "each", "few", "more",
              "other", "some", "such", "only", "own",  "so", "than", "s", "t", "can", "will", "just", "don", "now",
              "br", "39", "quot", "one", "seen", "ve", "film", "movie", "story", "one", "character", "time", "movies",
              "character","thing","people", "make"]

def clean_data(text):
  """
    processes the given string:
    - convert to lower case
    - remove punctuation
    - remove stop words
    - lemmatize
    :text: string which is the content of a comment file
    :return: processed string
  """
  lower_case = text.lower()
  wh_special_char = lower_case.translate(str.maketrans(string.punctuation,' '*len(string.punctuation)))
  cleaned_data = ""
  for word in wh_special_char.split():
    if word not in stop_words:
      cleaned_data += " " + lemmatizer.lemmatize(word)
  return cleaned_data

def print_metrics(prediction, labels):
  N = len(labels)
  count = {"P":0, "N":0, "Z":0}
  labelCount = {"P":0, "N":0, "Z":0}
  tp = {"P":0, "N":0, "Z":0}
  tn = {"P":0, "N":0, "Z":0}
  classes = ["P", "N", "Z"]
  for i in range(N):
    count[prediction[i]] += 1
    labelCount[labels[i]] += 1
    if prediction[i] == labels[i]:
      tp[prediction[i]] += 1
    for c in classes:
      if c != prediction[i] and c != labels[i]:
        tn[c] += 1
  accuracy = {"P":0, "N":0, "Z":0}
  precision = {"P":0, "N":0, "Z":0}
  recall = {"P":0, "N":0, "Z":0}
  print("\nOverall Accuracy: " + str(sum(tp.values())/N))
  for i in ["P", "N", "Z"]:
    if count[i] == 0:
      precision[i] = 1
    else:
      precision[i] = tp[i]/count[i]
    if labelCount[i] == 0:
      recall[i] = 1
    else:
      recall[i] = tp[i]/labelCount[i]
    accuracy[i] = (tp[i] + tn[i])/N
  print("\nClass Accuracies:")
  print(accuracy)
  print("\nClass Precisions:")
  print(precision)
  print("\nClass Recalls:")
  print(recall)
  print("\nMacro average accuracy: ")
  print(sum(accuracy.values()) / 3)
  print("\nMacro average precision: ")
  print(sum(precision.values()) / 3)
  print("\nMacro average recall: ")
  print(sum(recall.values()) / 3)

#def get_word_clouds(data_p, data_n, data_z):
 # word cloud images for insight into data
  # negative
#  word_cloud = WordCloud(collocation_threshold=30).generate(text = data_n)
#  plt.imshow(word_cloud, interpolation= "bilinear")
#  plt.savefig("negative_wordcloud.png")
#  # positive
#  word_cloud = WordCloud(collocation_threshold=30).generate(text = data_p)
#  plt.imshow(word_cloud, interpolation= "bilinear")
#  plt.savefig("positive_wordcloud.png")
  # neutral
#  word_cloud = WordCloud(collocation_threshold=30).generate(text = data_z)
#  plt.imshow(word_cloud, interpolation= "bilinear")
#  plt.savefig("neutral_wordcloud.png")

# word cloud for each class using bigrams
# change
#get_word_clouds(data_p, data_n, data_z)

#read the data
data = []
labels=[]
# variables to store data for word cloud
data_z = data_n = data_p = ""
FILE_PATH = "TRAIN/"
dirs = os.listdir(FILE_PATH)
dirsSorted = sorted(dirs, key=lambda filename:int(filename[:-6]))
for file in dirsSorted:
  filename=os.path.join(FILE_PATH,file)
  f = open(filename, "r", encoding="latin-1")
  text = f.read()
  text = clean_data(text)
  data.append(text)
  if file[-5] == "N":
    data_n += text
  elif file[-5] == "Z":
    data_z += text
  elif file[-5] == "P":
    data_p += text
  labels.append(file[-5])
  f.close()

#TF-IDF + SVM
vectorizer = TfidfVectorizer()
features = vectorizer.fit_transform(data)

# initiate model
clf = svm.SVC()

# train
clf.fit(features, labels)

# get the training accuracy
prediction = clf.predict(features)
print_metrics(prediction, labels)
with open("step3_model_Solis.pkl", "wb") as file:
  pickle.dump(vectorizer, file)
  pickle.dump(clf, file)

# Other methods
# We used three methods to vectorize
# 1) Count Vectorizer (binary option being True or False),  2) Tf Idf Vectorizer, 3) First two pipelined
# and three models
# 1) Naive Bayes, 2) Logistic Regression, 3) SVM
# (not all combinations)

### bag of words

# uncomment following two lines if you want to try a model with Count
#count = CountVectorizer()
#bag = count.fit_transform(data)

# use binary parameter for binary count (1 if word is in document, 0 otherwise)
# uncomment following two lines if you want to try a model with Binary Count
#count_bin = CountVectorizer(binary=True)
#bag_bin = count.fit_transform(data)

# now we can use the vectorized data (bag/bag_bin) with different models

# Naive Bayes + Count
#clf = MultinomialNB()
#clf.fit(bag.toarray(), labels)
#prediction = clf.predict(bag.toarray())
#print_metrics(prediction, labels)

# Naive Bayes + Binary Count
#clf = MultinomialNB()
#clf.fit(bag_bin.toarray(), labels)
#prediction = clf.predict(bag_bin.toarray())
#print_metrics(prediction, labels)

# SVM + Count
#clf = svm.SVC()
#clf.fit(bag.toarray(), labels)
#prediction = clf.predict(bag.toarray())
#print_metrics(prediction)

# Naive Bayes + Count + TfIdf
#transformVectorizer = TfidfTransformer()
#transformed = transformVectorizer.fit_transform(bag)
#clf = MultinomialNB()
#clf.fit(transformed, labels)
#prediction = clf.predict(transformed)
#print_metrics(prediction)

# Naive Bayes + Binary Count + TfIdf
#transformVectorizer = TfidfTransformer()
#transformed = transformVectorizer.fit_transform(bag_bin)
#clf = MultinomialNB()
#clf.fit(transformed, labels)
#prediction = clf.predict(transformed)
#print_metrics(prediction)

# Logistic Regression + TfIdf
#clf = LogisticRegression(random_state=0, solver='lbfgs')
#clf.fit(features, labels)
#prediction = clf.predict(features)
#print_metrics(prediction)

# SVM + TfIdf
#clf = svm.SVC()
#clf.fit(features, labels)
#prediction = clf.predict(features)
#print_metrics(prediction

#word2vec + SVM

#import spacy
# Load the spacy model that you have installed
#nlp = spacy.load('en_core_web_sm')

#features = []
#for i in data:
#    doc = nlp(i)
#    features.append(doc.vector)

# initiate model
#clf = svm.SVC(kernel='rbf', C=15)

# train
#clf.fit(features, labels)

# get the training accuracy
#prediction = clf.predict(features)
#print_metrics(prediction, labels)

# In[ ]:


from lib.AuroraExtension import AuroraExtension
import time
import datetime
import numpy as np
import pandas as pd
import cv2

class Aurora_Rainbow(AuroraExtension):
    def __init__(self, NeoPixels, HDMI):
        super().__init__(NeoPixels, HDMI)
        self.Author = "Andrew MacPherson (@AndrewMohawk)"
        self.Description = "This extension displays a rainbow pattern on the LEDs, a lot of this code is from https://learn.adafruit.com/circuitpython-essentials/circuitpython-neopixel"
        self.Name = "Aurora Rainbow Display ( LED Only )"
        self.count = 0
        self.current_frame = False
        self.noHDMI = True

    def takeScreenShot(self, filepath):
        # We have no screenshot since... well its just LEDs
        return True

    def wheel(self, pos):
        # Input a value 0 to 255 to get a color value.
        # The colours are a transition r - g - b - back to r.
        if pos < 0 or pos > 255:
            return (0, 0, 0)
        if pos < 85:
            return (255 - pos * 3, pos * 3, 0)
        if pos < 170:
            pos -= 85
            return (0, 255 - pos * 3, pos * 3)
        pos -= 170
        return (pos * 3, 0, 255 - pos * 3)

    def rainbow_cycle(self, j):
        for i in range(self.pixelsCount):
            rc_index = (i * 256 // self.pixelsCount) + j
            self.pixels[i] = self.wheel(rc_index & 255)
        self.getFrame(False)  # hack to have 'FPS'
        self.pixels.show()

    def visualise(self):

        # visualise!
        self.count += 1
        self.rainbow_cycle(self.count)
        if self.count == 255:
            self.count = 0

        time.sleep(0.01)


from wagtail.core.models import Page

def is_donation_page(page_id):
    from .models import CampaignPage, LandingPage   # Avoid circular import
    try:
        page = Page.objects.live().get(pk=page_id).specific
    except Page.DoesNotExist:
        return False

    return page.__class__ in [CampaignPage, LandingPage]

def queue_ga_event(request, event_data):
    if 'ga_events' in request.session:
        request.session['ga_events'].append(event_data)
        request.session.modified = True
    else:
        request.session['ga_events'] = [event_data]

def queue_datalayer_event(request, event_data):
    request.session['datalayer_event'] = event_data
    request.session.modified = True


from abc import ABCMeta, abstractmethod

import numpy as np
from scipy import integrate

class BaseDiscretizer(ABCMeta):
    """
    Base class for the discretizer classes in pgmpy. The discretizer
    classes are used to discretize a continuous random variable
    distribution into discrete probability masses.

    Parameters
    ----------
    factor: A ContinuousNode or a ContinuousFactor object
        the continuous node or factor representing the distribution
        to be discretized.

    low, high: float
        the range over which the function will be discretized.

    cardinality: int
        the number of states required in the discretized output.

    Examples
    --------
    >>> from scipy.stats import norm
    >>> from pgmpy.factors.continuous import ContinuousNode
    >>> normal = ContinuousNode(norm(0, 1).pdf)
    >>> from pgmpy.discretize import BaseDiscretizer
    >>> class ChildDiscretizer(BaseDiscretizer):
    ...     def get_discrete_values(self):
    ...         pass
    >>> discretizer = ChildDiscretizer(normal, -3, 3, 10)
    >>> discretizer.factor
    <pgmpy.factors.continuous.ContinuousNode.ContinuousNode object at 0x04C98190>
    >>> discretizer.cardinality
    10
    >>> discretizer.get_labels()
    ['x=-3.0', 'x=-2.4', 'x=-1.8', 'x=-1.2', 'x=-0.6', 'x=0.0', 'x=0.6', 'x=1.2', 'x=1.8', 'x=2.4']

    """

    def __init__(self, factor, low, high, cardinality):
        self.factor = factor
        self.low = low
        self.high = high
        self.cardinality = cardinality

    @abstractmethod
    def get_discrete_values(self):
        """
        This method implements the algorithm to discretize the given
        continuous distribution.

        It must be implemented by all the subclasses of BaseDiscretizer.

        Returns
        -------
        A list of discrete values or a DiscreteFactor object.
        """
        pass

    def get_labels(self):
        """
        Returns a list of strings representing the values about
        which the discretization method calculates the probability
        masses.

        Default value is the points -
        [low, low+step, low+2*step, ......... , high-step]
        unless the method is overridden by a subclass.

        Examples
        --------
        >>> from pgmpy.factors import ContinuousNode
        >>> from pgmpy.discretize import BaseDiscretizer
        >>> class ChildDiscretizer(BaseDiscretizer):
        ...     def get_discrete_values(self):
        ...         pass
        >>> from scipy.stats import norm
        >>> node = ContinuousNode(norm(0).pdf)
        >>> child = ChildDiscretizer(node, -5, 5, 20)
        >>> chld.get_labels()
        ['x=-5.0', 'x=-4.5', 'x=-4.0', 'x=-3.5', 'x=-3.0', 'x=-2.5',
         'x=-2.0', 'x=-1.5', 'x=-1.0', 'x=-0.5', 'x=0.0', 'x=0.5', 'x=1.0',
         'x=1.5', 'x=2.0', 'x=2.5', 'x=3.0', 'x=3.5', 'x=4.0', 'x=4.5']

        """
        step = (self.high - self.low) / self.cardinality
        labels = [
            f"x={str(i)}" for i in np.round(np.arange(self.low, self.high, step), 3)
        ]
        return labels

class RoundingDiscretizer(BaseDiscretizer):
    """
    This class uses the rounding method for discretizing the
    given continuous distribution.

    For the rounding method,

    The probability mass is,
    cdf(x+step/2)-cdf(x), for x = low

    cdf(x+step/2)-cdf(x-step/2), for low < x <= high

    where, cdf is the cumulative density function of the distribution
    and step = (high-low)/cardinality.

    Examples
    --------
    >>> import numpy as np
    >>> from pgmpy.factors.continuous import ContinuousNode
    >>> from pgmpy.factors.continuous import RoundingDiscretizer
    >>> std_normal_pdf = lambda x : np.exp(-x*x/2) / (np.sqrt(2*np.pi))
    >>> std_normal = ContinuousNode(std_normal_pdf)
    >>> std_normal.discretize(RoundingDiscretizer, low=-3, high=3,
    ...                       cardinality=12)
    [0.001629865203424451, 0.009244709419989363, 0.027834684208773178,
     0.065590616803038182, 0.120977578710013, 0.17466632194020804,
     0.19741265136584729, 0.17466632194020937, 0.12097757871001302,
     0.065590616803036905, 0.027834684208772664, 0.0092447094199902269]
    """

    def get_discrete_values(self):
        step = (self.high - self.low) / self.cardinality

        # for x=[low]
        discrete_values = [
            self.factor.cdf(self.low + step / 2) - self.factor.cdf(self.low)
        ]

        # for x=[low+step, low+2*step, ........., high-step]
        points = np.linspace(self.low + step, self.high - step, self.cardinality - 1)
        discrete_values.extend(
            [
                self.factor.cdf(i + step / 2) - self.factor.cdf(i - step / 2)
                for i in points
            ]
        )

        return discrete_values

class UnbiasedDiscretizer(BaseDiscretizer):
    """
    This class uses the unbiased method for discretizing the
    given continuous distribution.

    The unbiased method for discretization is the matching of the
    first moment method. It involves calculating the first order
    limited moment of the distribution which is done by the _lim_moment
    method.

    For this method,

    The probability mass is,
    (E(x) - E(x + step))/step + 1 - cdf(x), for x = low

    (2 * E(x) - E(x - step) - E(x + step))/step, for low < x < high

    (E(x) - E(x - step))/step - 1 + cdf(x), for x = high

    where, E(x) is the first limiting moment of the distribution
    about the point x, cdf is the cumulative density function
    and step = (high-low)/cardinality.

    References
    ----------
    Klugman, S. A., Panjer, H. H. and Willmot, G. E.,
    Loss Models, From Data to Decisions, Fourth Edition,
    Wiley, section 9.6.5.2 (Method of local moment matching) and
    exercise 9.41.

    Examples
    --------
    >>> import numpy as np
    >>> from pgmpy.factors import ContinuousNode
    >>> from pgmpy.factors.continuous import UnbiasedDiscretizer
    # exponential distribution with rate = 2
    >>> exp_pdf = lambda x: 2*np.exp(-2*x) if x>=0 else 0
    >>> exp_node = ContinuousNode(exp_pdf)
    >>> exp_node.discretize(UnbiasedDiscretizer, low=0, high=5, cardinality=10)
    [0.39627368905806137, 0.4049838434034298, 0.13331784003148325,
     0.043887287876647259, 0.014447413395300212, 0.0047559685431339703,
     0.0015656350182896128, 0.00051540201980112557, 0.00016965346326140994,
     3.7867260839208328e-05]

    """

    def get_discrete_values(self):
        lev = self._lim_moment
        step = (self.high - self.low) / (self.cardinality - 1)

        # for x=[low]
        discrete_values = [
            (lev(self.low) - lev(self.low + step)) / step
            + 1
            - self.factor.cdf(self.low)
        ]

        # for x=[low+step, low+2*step, ........., high-step]
        points = np.linspace(self.low + step, self.high - step, self.cardinality - 2)
        discrete_values.extend(
            [(2 * lev(i) - lev(i - step) - lev(i + step)) / step for i in points]
        )

        # for x=[high]
        discrete_values.append(
            (lev(self.high) - lev(self.high - step)) / step
            - 1
            + self.factor.cdf(self.high)
        )

        return discrete_values

    def _lim_moment(self, u, order=1):
        """
        This method calculates the kth order limiting moment of
        the distribution. It is given by -

        E(u) = Integral (-inf to u) [ (x^k)*pdf(x) dx ] + (u^k)(1-cdf(u))

        where, pdf is the probability density function and cdf is the
        cumulative density function of the distribution.

        Reference
        ---------
        Klugman, S. A., Panjer, H. H. and Willmot, G. E.,
        Loss Models, From Data to Decisions, Fourth Edition,
        Wiley, definition 3.5 and equation 3.8.

        Parameters
        ----------
        u: float
            The point at which the moment is to be calculated.

        order: int
            The order of the moment, default is first order.
        """

        def fun(x):
            return np.power(x, order) * self.factor.pdf(x)

        return integrate.quad(fun, -np.inf, u)[0] + np.power(u, order) * (
            1 - self.factor.cdf(u)
        )

    def get_labels(self):
        labels = list(
            f"x={str(i)}"
            for i in np.round(np.linspace(self.low, self.high, self.cardinality), 3)
        )
        return labels


import json
import logging
from time import sleep
from typing import Dict, List

from terragraph.ctf.tests.test_iot.test_node_software_upgrade import TestIotNSU1

LOG = logging.getLogger(__name__)

class TestNetworkIotNSU2(TestIotNSU1):
    TEST_NAME = "NetworkIOTv2-NSU-2"
    DESCRIPTION = "Upgrade network excluding specified link and node which are down using OEM's E2E controller with BitTorrent."

    def build_test_steps(self, fail_during_upgrade: bool = False) -> List[Dict]:
        """ """
        steps = []
        ignition_timeout_s = self.test_args["ignition_timeout_s"]
        upgrade_timeout_s = self.test_args["upgrade_timeout_s"]

        # read test data configs
        test_data = json.load(open(self.test_args["test_data"]))
        node_to_fail = test_data["fail_node"]
        link_to_fail = test_data["fail_link"]
        expected_offline_network = test_data["expected_offline_network"]
        skip_nodes = expected_offline_network["nodes"]
        skip_links = expected_offline_network["links"]
        if "wait_to_fail_time" in self.test_args.keys():
            wait_to_fail_time = self.test_args["wait_to_fail_time"]
        else:
            wait_to_fail_time = 0

        # upgrade controller if x86_image is given
        if self.test_args["x86_image_path"]:
            steps.append(
                {
                    "name": "Push x86 image",
                    "function": self.push_x86_image,
                    "function_args": (self.test_args["x86_image_path"],),
                    "success_msg": "x86 image was upgraded.",
                }
            )
        steps.extend(
            [
                self.get_common_test_steps()["check_software_versions"],
                self.get_common_x86_test_steps()["setup_x86_services"],
                # upload tg image to controller
                {
                    "name": "Push Puma image to x86 host",
                    "function": self.push_tg_image_to_x86,
                    "function_args": (
                        self.test_args["upload_image_path"],
                        "tg-image.bin",
                    ),
                    "success_msg": "Puma image was uploaded.",
                },
                # upload oem image to controller
                {
                    "name": "Push OEM image to x86 host",
                    "function": self.push_tg_image_to_x86,
                    "function_args": (
                        self.test_args["upload_oem_image_path"],
                        "oem-image.bin",
                    ),
                    "success_msg": "OEM image was uploaded.",
                },
                # start bitTorrent tracker service
                self.get_common_x86_test_steps()["start_x86_bt_tracker"],
                self.get_common_x86_test_steps()["start_x86_services"],
                {
                    "name": "Wait 5 seconds for controller to initialize",
                    "function": sleep,
                    "function_args": (5,),
                    "success_msg": "Finished waiting",
                },
                {
                    "name": "Verify PUMA image is seeding",
                    "function": self._verify_tg_image_seeding,
                    "function_args": (self.test_args["upload_image_path"],),
                    "success_msg": "E2E controller is seeding the Puma image.",
                },
                {
                    "name": "Check if network is ignited",
                    "function": self.try_until_timeout,
                    "function_args": (
                        self.controller_verify_topology_up,
                        (self.api_service_request, ["getTopology"]),
                        5,
                        ignition_timeout_s,
                    ),
                    "success_msg": "Network is up",
                    "error_handler": self.get_common_error_handler(),
                },
                {
                    "name": "Wait 60 seconds for last node to apply node config overrides changes",
                    "function": sleep,
                    "function_args": (60,),
                    "success_msg": "Finished waiting",
                },
                {
                    "name": "ReCheck network after node overrides take effect",
                    "function": self.try_until_timeout,
                    "function_args": (
                        self.controller_verify_topology_up,
                        (self.api_service_request, ["getTopology"]),
                        5,
                        120,
                    ),
                    "success_msg": "Network is up",
                    "error_handler": self.get_common_error_handler(),
                },
                # get tg nodes in network expected to be online for upgrading
                {
                    "name": "Get puma nodes in network expected to be online",
                    "function": self._get_puma_nodes,
                    "function_args": (skip_nodes,),
                    "success_msg": "retrieved puma nodes",
                },
                {
                    "name": f"fail tg link {link_to_fail['name']} and node {node_to_fail['name']} before upgrade",
                    "function": self.fail_link_and_node,
                    "function_args": (
                        link_to_fail,
                        node_to_fail,
                    ),
                    "concurrent": fail_during_upgrade,
                    "delay": wait_to_fail_time,
                    "success_msg": "sucesfully brought down tg link and node",
                },
                {
                    "name": "Start upgrade of online puma nodes in network",
                    "function": self._upgrade_nodes,
                    "function_args": (
                        upgrade_timeout_s,
                        self.puma_nodes,
                    ),
                    "concurrent": fail_during_upgrade,
                    "success_msg": "Upgrade request for puma nodes was sent.",
                },
                {
                    "name": "Check that all online puma nodes are upgraded",
                    "function": self.try_until_timeout,
                    "function_args": (
                        self._check_nodes_upgrade_status,
                        ([self.puma_nodes]),
                        5,
                        upgrade_timeout_s,
                    ),
                    "success_msg": "Puma nodes upgraded successfully.",
                    "error_handler": self.get_common_error_handler(),
                },
                # upgrade oem node in network after puma nodes complete upgrade
                {
                    "name": "Verify OEM image is seeding",
                    "function": self._verify_tg_image_seeding,
                    "function_args": (self.test_args["upload_oem_image_path"],),
                    "success_msg": "E2E controller is seeding the OEM image.",
                },
                {
                    "name": "Start upgrade of oem node",
                    "function": self._upgrade_nodes,
                    "function_args": (
                        upgrade_timeout_s,
                        [self.test_args["oem_node_name"]],
                    ),
                    "success_msg": "Upgrade request was sent for OEM node",
                },
                {
                    "name": "Check that OEM node is upgraded",
                    "function": self.try_until_timeout,
                    "function_args": (
                        self._check_nodes_upgrade_status,
                        ([self.test_args["oem_node_name"]]),
                        5,
                        upgrade_timeout_s,
                    ),
                    "success_msg": "OEM node is upgraded successfully",
                    "error_handler": self.get_common_error_handler(),
                },
                {
                    "name": "Check if expected nodes and links are up after upgrade",
                    "function": self.try_until_timeout,
                    "function_args": (
                        self.controller_verify_topology_up,
                        (
                            self.api_service_request,
                            ["getTopology"],
                            skip_nodes,
                            skip_links,
                        ),
                        5,
                        ignition_timeout_s,
                    ),
                    "success_msg": "Expected nodes and links are up after upgrade",
                    "error_handler": self.get_common_error_handler(),
                },
            ]
        )
        return steps

    def fail_link_and_node(self, link_to_fail: Dict, node_to_fail: Dict):
        """
        bring down link specified in link_to_fail.
        stop minion on node specified in node_to_fail
        """

        link_name = link_to_fail["name"]
        initiator_mac = link_to_fail["initiator_mac"]
        responder_mac = link_to_fail["responder_mac"]

        # disable auto ignition on link
        self.api_set_link_ignition_state(link_name, False)

        # dissoc link and wait 40s before checking link state in controller
        self.api_force_dissoc(initiator_mac, responder_mac)
        sleep(40)

        # check link is down
        self.api_check_link_state(link_name, False)

        # stop minion on node and wait 40s before checking node state in controller
        self.tg_restart_minion([node_to_fail["id"]], "stop")
        sleep(40)

        # check node is offline
        self.api_check_node_state(node_to_fail["name"], False)

    def get_test_steps(self) -> List[Dict]:

        # Get steps for software upgrade where node and link fail before upgrade begins
        steps = self.build_test_steps()
        return steps

class TestNetworkIotNSU3(TestNetworkIotNSU2):
    TEST_NAME = "NetworkIOTv2-NSU-3"
    DESCRIPTION = "Upgrade network while specified link and node are brought down, using OEM's E2E controller with BitTorrent."

    @staticmethod
    def test_params() -> Dict[str, Dict]:
        test_params: Dict[str, Dict] = super(
            TestNetworkIotNSU3, TestNetworkIotNSU3
        ).test_params()
        test_params["wait_to_fail_time"] = {
            "desc": (
                "time to wait after upgrade is started before failing node and link in network"
            ),
            "convert": int,
            "default": 120,
        }
        return test_params

    def get_test_steps(self) -> List[Dict]:

        # Get steps for software upgrade when node and link fail during the upgrade process
        steps = self.build_test_steps(True)
        return steps


import openai as openai

from SmartFriend.helpers import MAKE_SUMMARY, OPENING_TEXT, BONUS_CONDITION, FALLBACK_TEXT

def fallback_for_ai(response, conversation):
    if "ai" in response.lower():
        conversation.append(
            {
                "role": "system",
                "content": FALLBACK_TEXT,
            }
        )

        response = openai.ChatCompletion.create(
            model="gpt-3.5-turbo",
            messages=conversation,
        )

        return response.choices[0].message.content

    return response

def get_response(conversation):
    response = openai.ChatCompletion.create(
        model="gpt-3.5-turbo",
        messages=conversation,
    )

    response = fallback_for_ai(
        response.choices[0].message.content,
        conversation
    )

    return response

def start_conversation(summary):
    conversation = [
        {
            'role': 'system',
            'content': BONUS_CONDITION + " " + summary,
        },
        {
            'role': 'system',
            'content': OPENING_TEXT,
        },
    ]

    return conversation

def post_summary(conversation):
    if conversation:
        result = {
            'role': 'system',
            'content': MAKE_SUMMARY
        }

        return result

    return ""


import pandas as pd
from collections import defaultdict
import warnings
import visualize
import neat
import multiprocessing
import os
import matplotlib.pyplot as plt

import numpy as np

from sklearn.datasets import load_digits

os.environ["PATH"] += os.pathsep + 'C:/Program Files (x86)/Graphviz2.38/bin/'

digits = load_digits()

X = digits.images.reshape(
    digits.images.shape[0], digits.images.shape[1] * digits.images.shape[2])
y = digits.target

def softmax(x, axis=1):
    # 计算每行的最大值
    row_max = x.max(axis=axis)

    # 每行元素都需要减去对应的最大值，否则求exp(x)会溢出，导致inf情况
    row_max = row_max.reshape(-1, 1)
    x = x - row_max

    # 计算e的指数次幂
    x_exp = np.exp(x)
    x_sum = np.sum(x_exp, axis=axis, keepdims=True)
    s = x_exp / x_sum
    return s

def eval_genomes(genomes, config):
    for genome_id, genome in genomes:
        net = neat.nn.FeedForwardNetwork.create(genome, config)

        outputs = []
        for xi in x_inputs:
            output = net.activate(xi)
            outputs.append(output)

        px_outputs = softmax(np.array(outputs).reshape(
            samplesize, number_of_classification), axis=1)
        # the index of maximum in each line
        pred_outputs = np.argmax(px_outputs, axis=1)
        # replace index to real number, from higher number to lower number!!
        for i in range(number_of_classification):
            index = number_of_classification - i - 1
            pred_outputs[pred_outputs == index] = numbers[index]

        real_outputs = np.array(x_outputs).reshape(samplesize,)

        acc = np.sum(pred_outputs == real_outputs)/samplesize

        genome.fitness = acc

numbers = [*range(10)]  # number 2 > number 1
number_of_classification = len(numbers)
number_of_sampling = 100

list_y = y.tolist()
digits_indexes = []
for digit in numbers:
    li = [i for i in range(len(list_y)) if list_y[i] == digit]
    digits_indexes.extend(li[:number_of_sampling])

samplesize = X[digits_indexes].shape[0]
x_inputs = [tuple(c) for c in X[digits_indexes].tolist()]
x_outputs = [tuple(c)
             for c in y[digits_indexes].reshape(samplesize, 1).tolist()]

def run(config_file):
    # Load configuration.
    config = neat.Config(
        neat.DefaultGenome,
        neat.DefaultReproduction,
        neat.DefaultSpeciesSet,
        neat.DefaultStagnation,
        config_file,
    )

    # Create the population, which is the top-level object for a NEAT run.
    p = neat.Population(config)

    # add a stdout reporter to show progress in the terminal
    reporter = neat.StdOutReporter(False)
    p.add_reporter(reporter)
    stats = neat.StatisticsReporter()
    p.add_reporter(stats)
    #checkpointer = neat.Checkpointer(100)
    # p.add_reporter(checkpointer)
    # Run for up to 300 generations.
    winner = p.run(eval_genomes, 1000)

    return [stats, winner]

samplesize = len(x_outputs)

local_dir = os.getcwd()
config_path = os.path.join(
    local_dir, "config-feedforward-mnist"+str(number_of_classification))
[stats, winner] = run(config_path)

config = neat.Config(
    neat.DefaultGenome,
    neat.DefaultReproduction,
    neat.DefaultSpeciesSet,
    neat.DefaultStagnation,
    config_path,
)
winner_net = neat.nn.FeedForwardNetwork.create(winner, config)

visualize.draw_net(config, winner, prune_unused=False)
visualize.plot_stats(stats, ylog=False, view=False)
visualize.plot_species(stats, view=False)


# -*- coding: utf-8 -*-

from flask import request

from ApplicationExample import create_app

app = create_app()

@app.before_request
def before_request_api():
    """
    PC端app拦截器
    :return:
    """
    print('app app app app')
    path = request.path
    print(path)
    return


import gamelib
import png
import diseño
import TP_3
from pila import Pila

ANCHO_GRILLA=400
ALTO_GRILLA=500

def paint_nuevo(ancho, alto):
    '''inicializa el deshacer del programa con una imagen vacía de ancho x alto'''
    paint=[]
    for i in range(alto):
        fila=[]
        for j in range(ancho):
            fila.append("#FFFFFF")
        paint.append(fila)
    return paint

def copiar_paint(paint):
    paint_copia=paint_nuevo(len(paint[0]),len(paint))
    for i in range(len(paint)):
        for j in range(len(paint[0])):
            paint_copia[i][j]=paint[i][j]
    return paint_copia

def png_aux(paint):
    contador=6
    colores={
        "#FFFFFF":0,
        "#000000":1,
        "#0000FF":2,
        "#FFFF00":3,
        "#FF0000":4,
        "#00FF00":5
    }
    paleta = [
    (255, 255, 255),
    (0, 0, 0),
    (0, 0, 255),
    (255, 255, 0),
    (255,0,0),
    (0,255,0)
    ]
    paint_aux=paint_nuevo(len(paint[0]),len(paint))
    for i in range(len(paint)):
        for j in range(len(paint[0])):
            if paint[i][j] in colores:
                paint_aux[i][j]=colores[paint[i][j]]
            else:
                colores[paint[i][j]]=contador
                contador+=1
                r,g,b=int(paint[i][j][1:3],16),int(paint[i][j][3:5],16),int(paint[i][j][5:7],16)
                paleta.append((r,g,b))
                paint_aux[i][j]=colores[paint[i][j]]
    return paint_aux,paleta

def guardar_png(paint):
    ruta=gamelib.input("¿Como quiere guardar la imagen?")
    paint_aux,paleta=png_aux(paint)
    png.escribir(f"{ruta}.png",paleta,paint_aux)

def guardar_ppm(paint):
    alto=len(paint)
    ancho=len(paint[0])
    while True:
        ruta = gamelib.input("Ingrese ruta")
        if ruta is None or ".ppm" not in ruta:
            gamelib.say("El archivo no es un PPM")
        else:
            break
    try:
        with open(ruta,"w")as arch:
            arch.write("P3"+"\n")
            arch.write(f'{alto} {ancho}'+"\n")
            arch.write("255"+"\n")
            for i in range(alto):
                for j in range(ancho):
                    arch.write(f'{int(paint[i][j][1:3],16)} {int(paint[i][j][3:5],16)} {int(paint[i][j][5:7],16)}'+"\n")
    except FileNotFoundError:
        gamelib.say("No existe el archivo")
    except TypeError:
        pass

def cargar_ppm(paint,deshacer,rehacer):
    while True:
        ruta = gamelib.input("Ingrese ruta")
        if not ruta is None and ".ppm" not in ruta :
            gamelib.say("El archivo no es un PPM")
            continue
        else:
            break
    try:
        with open(ruta, "r") as arch:
            next(arch)
            alto, ancho = arch.readline().split(" ")
            alto = int(alto)
            ancho = int(ancho)
            paint=paint_nuevo(ancho,alto)
            next(arch)
            for i in range(alto):
                for j in range(ancho):
                    colores = arch.readline().split()
                    r, g, b = f'{int(colores[0]):02x}',f'{int(colores[1]):02x}',f'{int(colores[2]):02x}'
                    paint[i][j]=f'#{r+g+b}'
    except FileNotFoundError:
        gamelib.say("No existe el archivo")
    except TypeError:
        pass
    while not rehacer.esta_vacia():
        rehacer.desapilar()
    while not deshacer.esta_vacia():
        deshacer.desapilar()
    #Elimine la apilacion en esta funcion
    return paint

def botones(paint,ev,deshacer,rehacer):
    if 450<=ev.y<=470 and 30<=ev.x<=120:
        gamelib.draw_rectangle(29,449,121,471,fill="black")
        diseño.boton_PNG()
        guardar_png(paint)
    elif 450<=ev.y<=470 and 150<=ev.x<=240:
        gamelib.draw_rectangle(149,449,241,471,fill="black")
        diseño.boton_gppm()
        guardar_ppm(paint)
    elif 450<=ev.y<=470 and 270<=ev.x<=360:
        gamelib.draw_rectangle(269,449,361,471,fill="black")
        diseño.boton_cppm()
        paint=cargar_ppm(paint,deshacer,rehacer)
    elif 410<=ev.y<=430 and 240<=ev.x<=260: #Nueva funcion
        #Agregue un mensaje para cuando el usuario apreta sobre el balde
        gamelib.say("Para utilizar el balde, utilice la rueda del raton")
    return paint

def verificar_color():
    hexadecimal="abcdefABCDEF0123456789"
    while True:
        color=gamelib.input("Ingrese color:")
        if color==None or len(color)!=6 or any(car not in hexadecimal for car in color):
            gamelib.say("Color incorrecto")
            continue
        else:
            break
    return color

def copiar_color(paint,fila,columna):
    return paint[fila][columna][1:]

def main():
    gamelib.title("AlgoPaint")
    gamelib.resize(ANCHO_GRILLA,ALTO_GRILLA)
    paint=paint_nuevo(4,4)
    color="ffffff"
    tecla="n"
    deshacer=Pila()
    rehacer=Pila()
    #Aca habia una apilacion que elimine
    while gamelib.loop(fps=15):
        diseño.paint_mostrar(paint)
        diseño.color_seleccionado(tecla,color)
        diseño.cuadros_colores(color)
        for ev in gamelib.get_events():
            if ev.type == gamelib.EventType.KeyPress:
                tecla=ev.key
                if tecla=="e":
                    color=verificar_color()
                    tecla=color
                elif tecla=="c":
                    tecla=color
                elif tecla=="z":
                    if not deshacer.esta_vacia():
                        paint=TP_3.deshacer_paint(deshacer,rehacer)
                #Agrego la tecla "n", para que el usuario pueda volver a pintar por lo menos con el color negro
                    tecla="n"
                elif tecla=="x":
                #Evite usar la tecla "y" para hacer rehacer. Era la misma que para el color amarillo
                    if not rehacer.esta_vacia():
                        paint=TP_3.rehacer_paint(rehacer,deshacer)
                    tecla="n"
            if ev.type == gamelib.EventType.ButtonPress:
                fila = ev.y // (400//len(paint))
                columna = ev.x // (400//len(paint[0]))
                if 0 <= fila < len(paint) and 0 <= columna < len(paint[0]) and ev.y<=400:
                    if ev.mouse_button==1:
                        #Vacio pila de rehacer y cambio el orden en donde apilo
                        while not rehacer.esta_vacia():
                            rehacer.desapilar()
                        paint_copia=copiar_paint(paint)
                        deshacer.apilar(paint_copia)
                        diseño.pintar(tecla,paint,fila,columna,color)
                    if ev.mouse_button==2: #El balde ahora es un evento de tipo 2 (rueda del mouse)
                        #Vacio pila de rehacer y cambio el orden en donde apilo
                        while not rehacer.esta_vacia():
                            rehacer.desapilar()
                        paint_copia=copiar_paint(paint)
                        deshacer.apilar(paint_copia)
                        TP_3.balde(tecla,paint,fila,columna)
                    if ev.mouse_button==3:
                        color=copiar_color(paint,fila,columna)
                        tecla=color
                paint=botones(paint,ev,deshacer,rehacer)

gamelib.init(main)


# coding: utf-8
import pprint
import re  # noqa: F401

import six
from enum import Enum
from schema.aws.kms.awsapicallviacloudtrail.AWSAPICallViaCloudTrailItem import AWSAPICallViaCloudTrailItem  # noqa: F401,E501
from schema.aws.kms.awsapicallviacloudtrail.Object import Object  # noqa: F401,E501
from schema.aws.kms.awsapicallviacloudtrail.RequestParameters import RequestParameters  # noqa: F401,E501
from schema.aws.kms.awsapicallviacloudtrail.UserIdentity import UserIdentity  # noqa: F401,E501

class AWSAPICallViaCloudTrail(object):

    _types = {
        'requestParameters': 'RequestParameters',
        'userIdentity': 'UserIdentity',
        'eventID': 'str',
        'awsRegion': 'str',
        'eventVersion': 'str',
        'responseElements': 'Object',
        'sourceIPAddress': 'str',
        'eventSource': 'str',
        'resources': 'list[AWSAPICallViaCloudTrailItem]',
        'userAgent': 'str',
        'readOnly': 'bool',
        'eventType': 'str',
        'requestID': 'str',
        'eventTime': 'datetime',
        'eventName': 'str'
    }

    _attribute_map = {
        'requestParameters': 'requestParameters',
        'userIdentity': 'userIdentity',
        'eventID': 'eventID',
        'awsRegion': 'awsRegion',
        'eventVersion': 'eventVersion',
        'responseElements': 'responseElements',
        'sourceIPAddress': 'sourceIPAddress',
        'eventSource': 'eventSource',
        'resources': 'resources',
        'userAgent': 'userAgent',
        'readOnly': 'readOnly',
        'eventType': 'eventType',
        'requestID': 'requestID',
        'eventTime': 'eventTime',
        'eventName': 'eventName'
    }

    def __init__(self, requestParameters=None, userIdentity=None, eventID=None, awsRegion=None, eventVersion=None, responseElements=None, sourceIPAddress=None, eventSource=None, resources=None, userAgent=None, readOnly=None, eventType=None, requestID=None, eventTime=None, eventName=None):  # noqa: E501
        self._requestParameters = None
        self._userIdentity = None
        self._eventID = None
        self._awsRegion = None
        self._eventVersion = None
        self._responseElements = None
        self._sourceIPAddress = None
        self._eventSource = None
        self._resources = None
        self._userAgent = None
        self._readOnly = None
        self._eventType = None
        self._requestID = None
        self._eventTime = None
        self._eventName = None
        self.discriminator = None
        self.requestParameters = requestParameters
        self.userIdentity = userIdentity
        self.eventID = eventID
        self.awsRegion = awsRegion
        self.eventVersion = eventVersion
        self.responseElements = responseElements
        self.sourceIPAddress = sourceIPAddress
        self.eventSource = eventSource
        self.resources = resources
        self.userAgent = userAgent
        self.readOnly = readOnly
        self.eventType = eventType
        self.requestID = requestID
        self.eventTime = eventTime
        self.eventName = eventName

    @property
    def requestParameters(self):

        return self._requestParameters

    @requestParameters.setter
    def requestParameters(self, requestParameters):

        self._requestParameters = requestParameters

    @property
    def userIdentity(self):

        return self._userIdentity

    @userIdentity.setter
    def userIdentity(self, userIdentity):

        self._userIdentity = userIdentity

    @property
    def eventID(self):

        return self._eventID

    @eventID.setter
    def eventID(self, eventID):

        self._eventID = eventID

    @property
    def awsRegion(self):

        return self._awsRegion

    @awsRegion.setter
    def awsRegion(self, awsRegion):

        self._awsRegion = awsRegion

    @property
    def eventVersion(self):

        return self._eventVersion

    @eventVersion.setter
    def eventVersion(self, eventVersion):

        self._eventVersion = eventVersion

    @property
    def responseElements(self):

        return self._responseElements

    @responseElements.setter
    def responseElements(self, responseElements):

        self._responseElements = responseElements

    @property
    def sourceIPAddress(self):

        return self._sourceIPAddress

    @sourceIPAddress.setter
    def sourceIPAddress(self, sourceIPAddress):

        self._sourceIPAddress = sourceIPAddress

    @property
    def eventSource(self):

        return self._eventSource

    @eventSource.setter
    def eventSource(self, eventSource):

        self._eventSource = eventSource

    @property
    def resources(self):

        return self._resources

    @resources.setter
    def resources(self, resources):

        self._resources = resources

    @property
    def userAgent(self):

        return self._userAgent

    @userAgent.setter
    def userAgent(self, userAgent):

        self._userAgent = userAgent

    @property
    def readOnly(self):

        return self._readOnly

    @readOnly.setter
    def readOnly(self, readOnly):

        self._readOnly = readOnly

    @property
    def eventType(self):

        return self._eventType

    @eventType.setter
    def eventType(self, eventType):

        self._eventType = eventType

    @property
    def requestID(self):

        return self._requestID

    @requestID.setter
    def requestID(self, requestID):

        self._requestID = requestID

    @property
    def eventTime(self):

        return self._eventTime

    @eventTime.setter
    def eventTime(self, eventTime):

        self._eventTime = eventTime

    @property
    def eventName(self):

        return self._eventName

    @eventName.setter
    def eventName(self, eventName):

        self._eventName = eventName

    def to_dict(self):
        result = {}

        for attr, _ in six.iteritems(self._types):
            value = getattr(self, attr)
            if isinstance(value, list):
                result[attr] = list(map(
                    lambda x: x.to_dict() if hasattr(x, "to_dict") else x,
                    value
                ))
            elif hasattr(value, "to_dict"):
                result[attr] = value.to_dict()
            elif isinstance(value, dict):
                result[attr] = dict(map(
                    lambda item: (item[0], item[1].to_dict())
                    if hasattr(item[1], "to_dict") else item,
                    value.items()
                ))
            else:
                result[attr] = value
        if issubclass(AWSAPICallViaCloudTrail, dict):
            for key, value in self.items():
                result[key] = value

        return result

    def to_str(self):
        return pprint.pformat(self.to_dict())

    def __repr__(self):
        return self.to_str()

    def __eq__(self, other):
        if not isinstance(other, AWSAPICallViaCloudTrail):
            return False

        return self.__dict__ == other.__dict__

    def __ne__(self, other):
        return not self == other


import json
import os
import shutil
from . import diffusionai
from .diffusionai import ModelType
from py_src.osPath import get_cache_path, get_models_config_path, get_models_path
from enum import Enum

class DiffusersModel:
    def __init__(self, type: ModelType, path: str, name: str, description: str) -> None:
        self.type = type
        self.path = path
        self.name = name
        self.description = description

def get_model_type(string):
    for model_type in ModelType:
        if string == model_type.value:
            return model_type
    raise ValueError("Invalid model type")

def loadConfig():
    configPath = get_models_config_path()
    if os.path.exists(configPath):
        json_open = open(get_models_config_path(), 'r')
        json_load = json.load(json_open)
        data = []
        for json_data in json_load:
            if json_data['type'] == ModelType.SD_model.value:
                modelType = ModelType.SD_model
            elif json_data['type'] == ModelType.VAE_model.value:
                modelType = ModelType.VAE_model
            elif json_data['type'] == ModelType.LORA_model.value:
                modelType = ModelType.LORA_model
            else:
                raise ValueError('An unsupported model type was found.'
                                 'Please correct the contents of the file or delete it and try again.')

            data.append(DiffusersModel(
                type=modelType,
                path=json_data['path'],
                name=json_data['name'],
                description=json_data['description']
            ))
        return data
    else:
        if not os.path.isdir(get_models_path()):
            os.mkdir(get_models_path())
        with open(configPath, "w") as f:
            f.write('[]')
        return []

def idToName(config, id):
    return next((c.name for c in config if c.path == id), None)

def addNewModelToConfig(type: ModelType, path: str, name: str, description: str):
    json_open = open(get_models_config_path(), 'r')
    json_load = json.load(json_open)
    new_data = {
        "type": type.value,
        "path": path,
        "name": name,
        "description": description
    }
    if any(d['path'] == new_data['path'] for d in json_load):
        print('Data with path {} already exists. Skipping addition.'.format(
            new_data['path']))
    else:
        json_load.append(new_data)
        with open(get_models_config_path(), 'w') as f:
            json.dump(json_load, f)

def deleteModelConfig(path: str):
    with open(get_models_config_path(), 'r') as f:
        json_load = json.load(f)
    target_path =  [model for model in json_load if model['path'] == path]
    if len(target_path) == 0:
        raise ValueError('The specified path was not found in config')
    deleted_config = [model for model in json_load if model['path'] != path]
    with open(get_models_config_path(), 'w') as f:
        json.dump(deleted_config, f)
    diffusionai.AutoModelLoader(cache=get_models_path()).remove_model(target_path[0]['path'])

def updateModelConfig(path: str, name: str, description: str):
    json_open = open(get_models_config_path(), 'r')
    json_load = json.load(json_open)
    if len([model for model in json_load if model['path'] == path]) == 0:
        return False
    for item in json_load:
        if item['path'] == path:
            item['name'] = name
            item['description'] = description
            break
    with open(get_models_config_path(), 'w') as f:
        json.dump(json_load, f)
    return True


"""
Source: https://docs.python.org/2/library/sqlite3.html
This is an example implementation of a database.
In this program, a database will be initialized and filled with data.
Then, the data will be printed to the console.

Any changes to the database structure need to be committed into a new
database file. The old file will not accept changes to the structure of
data.
"""

import sqlite3

conn = sqlite3.connect("data_files/ex_database.db")
c = conn.cursor()

try:
    # Create table
    c.execute('''CREATE TABLE listings
              (sku INTEGER, title TEXT, price REAL)''')
except sqlite3.OperationalError:
    print("Skipping Creation of database file: Already Created.")

# # Never do this -- insecure!
# some_title = 'New Blue Shoes'
# c.execute("SELECT * FROM listings WHERE title = '%s'" % some_title)

# Do this instead
# t = ('RHAT',)
# c.execute('SELECT * FROM stocks WHERE symbol=?', t)
# print(c.fetchone())

# Larger example that inserts many records at a time
list_of_listings = [(1, 'Like-New Medium Blue Socks', 10.99),
                    (2, 'Used Small Lime Green Gloves', 11.99),
                    (3, 'New Medium Navy Blue Gloves', 12.99),
                    (4, 'Like-New Extra Large Brown Hat', 13.99),
                    ]

c.executemany('INSERT INTO listings VALUES (?,?,?)', list_of_listings)
conn.commit()  # Commit changes to database file.

for row in c.execute('SELECT * FROM listings ORDER BY price'):
    print(row)

conn.close()  # Close the connection to the database.


# pylint: skip-file
"""remove upload dir id from db

Revision ID: 83c0cf708bbd
Revises: 2c76c101e8eb
Create Date: 2020-05-18 16:34:10.195705

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '83c0cf708bbd'
down_revision = '2c76c101e8eb'
branch_labels = None
depends_on = None

def upgrade():
    op.drop_column('submission', 'upload_dir_id')

def downgrade():
    op.add_column('submission',
        sa.Column('upload_dir_id', sa.Integer)
    )


#!/usr/bin/env python

from flask import Flask, request, redirect, url_for, render_template
import rpyc

UPLOAD_FOLDER = '../uploads/'
ALLOWED_EXTENSIONS = set(['py', 'sh'])

app = Flask(__name__)
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
def allowed_file(filename):
    return '.' in filename and \
           filename.rsplit('.', 1)[1] in ALLOWED_EXTENSIONS

@app.route('/',methods=['GET', 'POST'])
def node_page():
    if request.method == 'POST':
        return 'posted'

    c = rpyc.connect('localhost',18861)
    info = c.root.exposed_get_node_info()
    load = c.root.exposed_get_load()
    return render_template('index.html', info=info, load=load)

"""
@app.route('/node_info/')
@app.route('/submit_job/')
@app.route('/network_load/')
@app.route('/projects/')

def projects():
    return render_template('projects.html')

@app.route('/about')
def about():
    return 'render_template('about.html')
"""

if __name__ == '__main__':
    app.debug = True
    app.run(host='192.168.1.101')


import cv2
import numpy as np
filename='chess.jpg'
img= cv2.imread(filename)
gray=cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
gray=np.float32(gray)
dst=cv2.cornerHarris(gray,2,3,0.04)
#dilate the result to mark the corners
#cv2.dilate(dst,	None)
# Threshold for an optimal value, it may vary depending on the image.
img[dst>0.01*dst.max()]=[0,0,255]
cv2.imshow('dst',img)
if cv2.waitKey(0) & 0xff == 27:
	cv2.destroyAllWindows()