import torch
import torch.nn as nn
import matplotlib.pyplot as plt
import torch.nn.functional as F
from sklearn import metrics
import seaborn as sns
from torch_sparse import spspmm
import numpy as np

from src.visualization.visualize import Visualization
from src.features.link_prediction import Link_prediction

class BDRRAA(nn.Module, Link_prediction, Visualization):
    def __init__(self, k, d, sample_size, data, data_type = "sparse", data2 = None, non_sparse_i = None, non_sparse_j = None, sparse_i_rem = None, sparse_j_rem = None):
        super(BDRRAA, self).__init__()
        self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
        self.data_type = data_type
        self.sample_size = sample_size
        self.k = k
        self.d = d

        self.sparse_i_idx = data.to(self.device)
        self.sparse_j_idx = data2.to(self.device)
        self.non_sparse_i_idx_removed = non_sparse_i.to(self.device)
        self.non_sparse_j_idx_removed = non_sparse_j.to(self.device)
        self.sparse_i_idx_removed = sparse_i_rem.to(self.device)
        self.sparse_j_idx_removed = sparse_j_rem.to(self.device)
        self.removed_i = torch.cat((self.non_sparse_i_idx_removed, self.sparse_i_idx_removed))
        self.removed_j = torch.cat((self.non_sparse_j_idx_removed, self.sparse_j_idx_removed))

        self.sample_shape = (len(self.sparse_i_idx), len(self.sparse_j_idx))
        self.sampling_i_weights = torch.ones(self.sample_shape[0], device = self.device)
        self.sampling_j_weights = torch.ones(self.sample_shape[1], device = self.device)

        self.sample_i_size = int(self.sample_shape[0] * self.sample_size)
        self.sample_j_size = int(self.sample_shape[1] * self.sample_size)

        self.beta = torch.nn.Parameter(torch.randn(self.sample_shape[0], device = self.device))
        self.gamma = torch.nn.Parameter(torch.randn(self.sample_shape[1], device = self.device))
        self.softplus = nn.Softplus()
        self.A = torch.nn.Parameter(torch.randn(self.d, self.k, device = self.device))

        self.Z_i = torch.nn.Parameter(torch.randn(self.k, self.sample_shape[0], device = self.device))
        self.Z_j = torch.nn.Parameter(torch.randn(self.k, self.sample_shape[1], device = self.device))

        self.Gate = torch.nn.Parameter(torch.randn(self.sample_shape[0] + self.sample_shape[1], self.k, device = self.device))

        self.losses = []
        self.N = self.sample_shape[0] + self.sample_shape[1]

        Link_prediction.__init__(self)
        Visualization.__init__(self)

    def sample_network(self):
        # USE torch_sparse lib i.e. : from torch_sparse import spspmm

        # sample for undirected network
        sample_i_idx = torch.multinomial(self.sampling_i_weights, self.sample_i_size, replacement=False)
        sample_j_idx = torch.multinomial(self.sampling_j_weights, self.sample_j_size, replacement=False)
        # translate sampled indices w.r.t. to the full matrix, it is just a diagonal matrix
        indices_i_translator = torch.cat([sample_i_idx.unsqueeze(0), sample_i_idx.unsqueeze(0)], 0)
        indices_j_translator = torch.cat([sample_j_idx.unsqueeze(0), sample_j_idx.unsqueeze(0)], 0)
        # adjacency matrix in edges format
        edges = torch.cat([self.sparse_i_idx.unsqueeze(0), self.sparse_j_idx.unsqueeze(0)], 0)
        # matrix multiplication B = Adjacency x Indices translator
        # see spspmm function, it give a multiplication between two matrices
        # indexC is the indices where we have non-zero values and valueC the actual values (in this case ones)
        indexC, valueC = spspmm(edges, torch.ones(edges.shape[1]), indices_j_translator,
                                torch.ones(indices_j_translator.shape[1]), self.sample_shape[0], self.sample_shape[1],
                                self.sample_shape[1], coalesced=True)
        # second matrix multiplication C = Indices translator x B, indexC returns where we have edges inside the sample
        indexC, valueC = spspmm(indices_i_translator, torch.ones(indices_i_translator.shape[1]), indexC, valueC,
                                self.sample_shape[0], self.sample_shape[0], self.sample_shape[1], coalesced=True)

        # edge row position
        sparse_i_sample = indexC[0, :]
        # edge column position
        sparse_j_sample = indexC[1, :]

        return sample_i_idx, sample_j_idx, sparse_i_sample, sparse_j_sample

    def log_likelihood(self):
        sample_i_idx, sample_j_idx, sparse_sample_i, sparse_sample_j = self.sample_network()
        Z_i = F.softmax(self.Z_i, dim=0)  # (K x N)
        Z_j = F.softmax(self.Z_j, dim=0)
        Z = torch.cat((Z_i[:,sample_i_idx], Z_j[:,sample_j_idx]),1) #Concatenate partition embeddings
        Gate = torch.cat((self.Gate[sample_i_idx,:], self.Gate[sample_j_idx,:]), 0)
        Gate = torch.sigmoid(Gate)  # Sigmoid activation function
        C = (Z.T * Gate) / (Z.T * Gate).sum(0)  # Gating function
        # For the nodes without links
        bias_matrix = self.beta[sample_i_idx].unsqueeze(1) + self.gamma[sample_j_idx]  # (N x N)
        AZC = torch.mm(self.A, torch.mm(Z, C))
        mat = (torch.exp(bias_matrix -
                         ((torch.mm(AZC,Z_i[:,sample_i_idx]).T.unsqueeze(1) -
                           torch.mm(AZC,Z_j[:,sample_j_idx]).T + 1e-06) ** 2).sum(-1) ** 0.5)).sum()
        mat_links = ((self.beta[sparse_sample_i] + self.gamma[sparse_sample_j]) -
                     (((AZC @ Z_i[:,sparse_sample_i]).T -
                       (AZC @ Z_j[:,sparse_sample_j]).T + 1e-06) ** 2).sum(-1) ** 0.5).sum()
        log_likelihood_sparse = mat_links - mat
        return log_likelihood_sparse

    def train(self, iterations, LR = 0.1, print_loss = False):
        optimizer = torch.optim.Adam(params = self.parameters(), lr=LR)

        for _ in range(iterations):
            loss = - self.log_likelihood() / self.N
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
            self.losses.append(loss.item())
            if print_loss:
                print('Loss at the',_,'iteration:',loss.item())

if __name__ == "__main__":
    seed = 42
    torch.random.manual_seed(seed)
    k = 3
    d = 2

    # Data
    dataset = "drug-gene"
    data = torch.from_numpy(np.loadtxt("../data/train_masks/" + dataset + "/sparse_i.txt")).long()
    data2 = torch.from_numpy(np.loadtxt("../data/train_masks/" + dataset + "/sparse_j.txt")).long()
    sparse_i_rem = torch.from_numpy(np.loadtxt("../data/train_masks/" + dataset + "/sparse_i_rem.txt")).long()
    sparse_j_rem = torch.from_numpy(np.loadtxt("../data/train_masks/" + dataset + "/sparse_j_rem.txt")).long()
    non_sparse_i = torch.from_numpy(np.loadtxt("../data/train_masks/" + dataset + "/non_sparse_i.txt")).long()
    non_sparse_j = torch.from_numpy(np.loadtxt("../data/train_masks/" + dataset + "/non_sparse_j.txt")).long()

    model = BDRRAA(k = k, d = d, sample_size = 0.2, data = data, data2 = data2, non_sparse_i=non_sparse_i, non_sparse_j=non_sparse_j, sparse_i_rem=sparse_i_rem, sparse_j_rem=sparse_j_rem)
    iterations = 10000
    model.train(interations = iterations, print_loss = True)
    # Plotting latent space
    Z_i = F.softmax(model.Z_i, dim=0)
    Z_j = F.softmax(model.Z_j, dim=0)
    Z = torch.cat((Z_i,Z_j),1)
    G = torch.sigmoid(model.Gate)
    C = (Z.T * G) / (Z.T * G).sum(0)

    embeddings = torch.matmul(model.A, torch.matmul(torch.matmul(Z, C), Z)).T
    archetypes = torch.matmul(model.A, torch.matmul(Z, C))

    fig, ([ax1, ax2]) = plt.subplots(nrows=1, ncols=2)
    sns.heatmap(Z.detach().numpy(), cmap="YlGnBu", cbar=False, ax=ax1)
    sns.heatmap(C.T.detach().numpy(), cmap="YlGnBu", cbar=False, ax=ax2)
    #sns.heatmap(Z_j.detach().numpy(), cmap="YlGnBu", cbar=False, ax=ax3)
    #sns.heatmap(C_j.T.detach().numpy(), cmap="YlGnBu", cbar=False, ax=ax4)

    if embeddings.shape[1] == 3:
        fig = plt.figure()
        ax = fig.add_subplot(projection='3d')
        ax.scatter(embeddings[:, 0].detach().numpy(), embeddings[:, 1].detach().numpy(),
                   embeddings[:, 2].detach().numpy(), c='red')
        ax.scatter(archetypes[0, :].detach().numpy(), archetypes[1, :].detach().numpy(),
                   archetypes[2, :].detach().numpy(), marker='^', c='black')
        '''ax.scatter(embeddings_j[:, 0].detach().numpy(), embeddings_j[:, 1].detach().numpy(),
                   embeddings_j[:, 2].detach().numpy(), c='blue')
        ax.scatter(archetypes_j[0, :].detach().numpy(), archetypes_j[1, :].detach().numpy(),
                   archetypes_j[2, :].detach().numpy(), marker='^', c='purple')'''
        ax.set_title(f"Latent space after {iterations} iterations")
    else:
        fig, (ax1, ax2) = plt.subplots(1, 2)
        ax1.scatter(embeddings[model.input_size[0]:, 0].detach().numpy(), embeddings[model.input_size[0]:, 1].detach().numpy(), c='red')
        ax1.scatter(embeddings[:model.input_size[0], 0].detach().numpy(), embeddings[:model.input_size[0], 1].detach().numpy(), c='blue')
        ax1.scatter(archetypes[0, :].detach().numpy(), archetypes[1, :].detach().numpy(), marker='^', c='black')
        #ax1.scatter(embeddings_j[:, 0].detach().numpy(), embeddings_j[:, 1].detach().numpy(), c='blue')
        #ax1.scatter(archetypes_j[0, :].detach().numpy(), archetypes_j[1, :].detach().numpy(), marker='^', c='purple')
        ax1.set_title(f"Latent space after {iterations} iterations")
        # Plotting learning curve
        ax2.plot(model.losses)
        ax2.set_yscale("log")
        ax2.set_title("Loss")
    plt.show()

    model.plot_auc()

"""
    def link_prediction(self):
        with torch.no_grad():
            Z_i = F.softmax(self.Z_i, dim=0)  # (K x N)
            Z_j = F.softmax(self.Z_j, dim=0)
            Z = torch.cat((Z_i, Z_j),1) #Concatenate partition embeddings
            #Z = F.softmax(Z, dim=0)
            G = F.sigmoid(self.G)
            C = (Z.T * G) / (Z.T * G).sum(0)  # Gating function

            M_i = torch.matmul(self.A, torch.matmul(torch.matmul(Z, C), Z[:, self.removed_i])).T  # Size of test set e.g. K x N
            M_j = torch.matmul(self.A, torch.matmul(torch.matmul(Z, C), Z[:, self.removed_j])).T
            z_pdist_test = ((M_i - M_j + 1e-06) ** 2).sum(-1) ** 0.5  # N x N
            theta = (self.beta[self.removed_i] + self.gamma[self.removed_j] - z_pdist_test)  # N x N

            # Get the rate -> exp(log_odds)
            rate = torch.exp(theta)  # N

            # TODO Skal lige ha tjekket om det er den rigtige rækkefølge.
            target = torch.cat((torch.zeros(self.non_sparse_i_idx_removed.shape[0]), torch.ones(self.sparse_i_idx_removed.shape[0])))

            fpr, tpr, threshold = metrics.roc_curve(target, rate.numpy())

            # Determining AUC score and precision and recall
            auc_score = metrics.roc_auc_score(target, rate.cpu().data.numpy())

            return auc_score, fpr, tpr

    G = mmread('data/toy_data/divorce/divorce.mtx')

    edge_list = torch.tensor([G.row,G.col]).T
    edge_list = edge_list.long()
    seed = 42
    torch.random.manual_seed(seed)

    # A = mmread("data/raw/soc-karate.mtx")
    # A = A.todense()
    k = 3
    d = 2

    link_pred = True

    if link_pred:
        num_samples = round(0.2 * ((50 * 9)))
        idx_i_test = torch.multinomial(input=torch.arange(0, float(50)), num_samples=num_samples,
                                       replacement=True)
        idx_j_test = torch.multinomial(input=torch.arange(0, float(9)), num_samples=num_samples, replacement=True)

        test = torch.stack((idx_i_test, idx_j_test))

        # TODO: could be a killer.. maybe do it once and save adjacency list ;)
        def if_edge(a, edge_list):
            a = a.tolist()
            edge_list = edge_list.tolist()
            a = list(zip(a[0], a[1]))
            edge_list = list(zip(edge_list[0], edge_list[1]))
            return [a[i] in edge_list for i in range(len(a))]

        target = [] #if_edge(test, edge_list)
        G = G.todense()
        for i in range(len(idx_i_test)):
            if G[idx_i_test[i], idx_j_test[i]] == 1:
                G[idx_i_test[i], idx_j_test[i]] = 0
                target.append(True)
            else:
                target.append(False)

        G = scipy.sparse.coo_matrix(G)
        edge_list = torch.tensor([G.row,G.col]).T
        edge_list = edge_list.long()

"""


import matplotlib.colors
import numpy as np
import time
import pandas as pd
import matplotlib.pyplot as plt
from scipy.ndimage import gaussian_filter
import tcod

np.random.seed(5)
tempType = "normal"
rainType = "normal"

def diamond_square(n, bias=False):
    # The array must be square with edge length 2**n + 1
    N = 2 ** n + 1

    arr = np.zeros((N, N))
    # f scales the random numbers at each stage of the algorithm
    f = 1.0
    if not bias:
        # Initialise the array with random numbers at its corners
        arr[0::N - 1, 0::N - 1] = np.random.uniform(-1, 1, (2, 2))
    else:
        arr[0] = np.random.uniform(-1.5, -.5, N)
        arr[N - 1] = np.random.uniform(.5, 1.5, N)

    side = N - 1
    nsquares = 1
    while side > 1:
        sideo2 = side // 2

        # Diamond step
        for ix in range(nsquares):
            for iy in range(nsquares):
                x0, x1, y0, y1 = ix * side, (ix + 1) * side, iy * side, (iy + 1) * side
                xc, yc = x0 + sideo2, y0 + sideo2
                # Set this pixel to the mean of its "diamond" neighbours plus
                # a random offset.
                arr[yc, xc] = (arr[y0, x0] + arr[y0, x1] + arr[y1, x0] + arr[y1, x1]) / 4
                arr[yc, xc] += f * np.random.uniform(-1, 1)

        # Square step: NB don't do this step until the pixels from the preceding
        # diamond step have been set.
        for iy in range(2 * nsquares + 1):
            yc = sideo2 * iy
            for ix in range(nsquares + 1):
                xc = side * ix + sideo2 * (1 - iy % 2)
                if not (0 <= xc < N and 0 <= yc < N):
                    continue
                tot, ntot = 0., 0
                # Set this pixel to the mean of its "square" neighbours plus
                # a random offset. At the edges, it has only three neighbours
                for (dx, dy) in ((-1, 0), (1, 0), (0, -1), (0, 1)):
                    xs, ys = xc + dx * sideo2, yc + dy * sideo2
                    if not (0 <= xs < N and 0 <= ys < N):
                        continue
                    else:
                        tot += arr[ys, xs]
                        ntot += 1
                arr[yc, xc] += tot / ntot + f * np.random.uniform(-1, 1)
        side = sideo2
        nsquares *= 2
        f /= 2

    #now we need to normalize the maps
    #get the min value
    minValue = 0
    for y in arr:
        if min(y) < minValue:
            minValue = min(y)
    arr = arr + (minValue * -1)
    maxValue = 1
    for y in arr:
        if max(y) > maxValue:
            maxValue = max(y)
    arr = arr/maxValue
    #add the negative to all the numbers to get the lowest value to 0
    #divide all the numbers by the maximum value
    #now it's between 0 and 1

    return arr

mapSize = 8
N = 2**mapSize+1

st = time.time()
heightMap = diamond_square(mapSize)
dt = round(time.time() - st, 1)
print("generated heightMap\t\t{} sec".format(dt))

st = time.time()
rainMap = diamond_square(mapSize)
dt = round(time.time() - st, 1)
print("generated rainMap\t\t{} sec".format(dt))

st = time.time()
tempMap = diamond_square(mapSize)
dt = round(time.time() - st, 1)
print("generated tempMap\t\t{} sec".format(dt))

def normalize_map(data):
    return (data - np.min(data)) / (np.max(data) - np.min(data))

#adjuting temperature in a different way
#add or subtract 1/10th of the distance from 0 or 1 (depending on if you're making it hotter or colder
def heat_world(temp_map, h):
    #h is the heat intensity modifier
    for y in range(len(temp_map)):
        for x in range(len(temp_map)):
            temp_map[y][x] += (1 - temp_map[y][x])/h
    return temp_map

def cool_world(temp_map, h):
    #h is the heat intensity modifier
    for y in range(len(temp_map)):
        for x in range(len(temp_map)):
            temp_map[y][x] -= (temp_map[y][x])/h
    return temp_map

def wet_world(rain_map, h):
    #h is the heat intensity modifier
    for y in range(len(rain_map)):
        for x in range(len(rain_map)):
            rain_map[y][x] += (1 - rain_map[y][x])/h
    return rain_map

def dry_world(rain_map, h):
    #h is the heat intensity modifier
    for y in range(len(rain_map)):
        for x in range(len(rain_map)):
            rain_map[y][x] -= (rain_map[y][x])/h
    return rain_map

heightMap = normalize_map(heightMap)

# as the height increases the temperature decreases
#tempMap = tempMap - (np.sqrt(heightMap)/2)

# there are 8 levels for rain
# fuck with the rain to get what we want
st = time.time()
rainMap = normalize_map(rainMap)

#rainMap = dry_world(rainMap, -3)
#rainMap = wet_world(rainMap, -2)

if rainType == "dry":
    rainSplines = [0.2, 0.3, 0.4, 0.5, 0.7, 0.8, 0.95]
elif rainType == "normal":
    #rainSplines = [0.05, 0.1, 0.3, 0.45, 0.6, 0.75, 0.9]
    rainSplines = [0.1, 0.15, 0.3, 0.45, 0.6, 0.8, 0.9]
elif rainType == "wet":
    rainSplines = [0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6]
for y in range(len(rainMap)):
    for x in range(len(rainMap)):
        if 0 <= rainMap[y][x] < rainSplines[0]:
            rainMap[y][x] = 0
        elif rainSplines[0] <= rainMap[y][x] < rainSplines[1]:
            rainMap[y][x] = 1
        elif rainSplines[1] <= rainMap[y][x] < rainSplines[2]:
            rainMap[y][x] = 2
        elif rainSplines[2] <= rainMap[y][x] < rainSplines[3]:
            rainMap[y][x] = 3
        elif rainSplines[3] <= rainMap[y][x] < rainSplines[4]:
            rainMap[y][x] = 4
        elif rainSplines[4] <= rainMap[y][x] < rainSplines[5]:
            rainMap[y][x] = 5
        elif rainSplines[5] <= rainMap[y][x] < rainSplines[6]:
            rainMap[y][x] = 6
        elif rainSplines[6] <= rainMap[y][x] :
            rainMap[y][x] = 7
rainMap = rainMap.astype(int)
dt = round(time.time() - st, 1)
print("splined rainMap\t\t\t{} sec".format(dt))

# there are 7 for temperature
st = time.time()
tempMap = normalize_map(tempMap)

#tempMap = heat_world(tempMap, -2)
#tempMap = cool_world(tempMap, -3)

if tempType == "hot":
    tempSplines = [0.05, 0.1, 0.2, 0.3, 0.4, 0.5]
elif tempType == "normal":
    #tempSplines = [0.05, 0.1, 0.2, 0.4, 0.6, 0.8]
    tempSplines = [0.1, 0.15, 0.2, 0.4, 0.8, 0.9]
elif tempType == "cold":
    tempSplines = [0.2, 0.3, 0.5, 0.7, 0.8, 0.95]
for y in range(len(tempMap)):
    for x in range(len(tempMap)):
        if 0 <= tempMap[y][x] < tempSplines[0]:
            tempMap[y][x] = 0
        elif tempSplines[0] <= tempMap[y][x] < tempSplines[1]:
            tempMap[y][x] = 1
        elif tempSplines[1] <= tempMap[y][x] < tempSplines[2]:
            tempMap[y][x] = 2
        elif tempSplines[2] <= tempMap[y][x] < tempSplines[3]:
            tempMap[y][x] = 3
        elif tempSplines[3] <= tempMap[y][x] < tempSplines[4]:
            tempMap[y][x] = 4
        elif tempSplines[4] <= tempMap[y][x] < tempSplines[5]:
            tempMap[y][x] = 5
        elif tempSplines[5] <= tempMap[y][x] :
            tempMap[y][x] = 6

tempMap = tempMap.astype(int)
dt = round(time.time() - st, 1)
print("splined tempMap\t\t\t{} sec".format(dt))

##############
#plt.imshow(heightMap, cmap='magma')
#plt.axis('off')
#plt.show()
#
#plt.imshow(rainMap, cmap='magma')
#plt.axis('off')
#plt.show()
#
#plt.imshow(tempMap, cmap='turbo')
#plt.axis('off')
#plt.show()
#
##############
st = time.time()
biomeGuide = [
    [1,  1,  1,  2,  2,  2,  2,  2],
    [1,  1,  1,  3,  3,  4,  4,  5],
    [6,  6,  7,  8,  8,  8,  8,  5],
    [6,  6,  7,  9,  9, 10, 11, 12],
    [13, 14, 15, 16, 17, 10, 11, 12],
    [13, 14, 15, 16, 18, 19, 20, 21],
    [13, 14, 22, 23, 24, 25, 25, 26]
]
#biome mapping
# 0   ocean
# 1   polar desert
# 2   ice cap
# 3   tundra
# 4   wet tundra
# 5   polar wetlands
# 6   cool desert
# 7   step
# 8   boreal forest
# 9   temperate woodlands
# 10  temperate forest
# 11  temperate wet forest
# 12  temperate wetlands
# 13  extreme desert
# 14  desert
# 15  subtropical scrub
# 16  subtropical wetlands
# 17  mediterranean
# 18  subtropical dry forest
# 19  subtropical forest
# 20  subtropical wet forest
# 21  subtropical wetlands
# 22  tropical scrub
# 23  tropical woodlands
# 24  tropical dry forest
# 25  tropical wet forest
# 26  tropical wetlands

seaLevel = np.median(heightMap)

biomeMap = np.zeros((N, N))
for y in range(len(biomeMap)):
    for x in range(len(biomeMap)):
        if heightMap[y][x] < seaLevel:
            biomeMap[y][x] = 0
        else:
            biomeMap[y][x] = biomeGuide[tempMap[y][x]][rainMap[y][x]]
dt = round(time.time() - st, 1)
print("generated biomeMap\t\t{} sec".format(dt))

np.savetxt("map_m3.csv", biomeMap, delimiter=",")

# generate cities
st = time.time()
cityMap = np.zeros((N,N))
numCities = N/2
while numCities > 0:
    rY = np.random.randint(0,N)
    rX = np.random.randint(0,N)
    if heightMap[rY][rX] > seaLevel:
        if not biomeMap[rY][rX] in [1, 2, 13]:
            cityMap[rY][rX] = 1
            numCities -= 1

# adjust the size
# pick a random city to add 1 pop to
# add that city back into the pool again
# get the list of cities
cityList = []
for y in range(len(cityMap)):
    for x in range(len(cityMap)):
        if cityMap[y][x] > 0:
            cityList.append([y, x])

worldPop = len(cityList)
for i in range(worldPop):
    cityNum = np.random.randint(0,len(cityList))
    cityToAdjust = cityList[cityNum]

    # this makes it zipfian
    # adjust based on temp and rainfall
    if biomeMap[cityToAdjust[0]][cityToAdjust[1]] in [0, 1, 2, 3, 4, 5, 6, 7, 13, 14, 15, 22, 26, 25, 21]:
        if np.random.uniform(0,1) < 1/4:
            cityMap[cityToAdjust[0]][cityToAdjust[1]] += 1
            cityList.append(cityToAdjust)
    else:
        cityMap[cityToAdjust[0]][cityToAdjust[1]] += 1
        cityList.append(cityToAdjust)

dt = round(time.time() - st, 1)
print("generated cityMap\t\t{} sec".format(dt))

np.savetxt("citymap_m3.csv", cityMap, delimiter=",")

st = time.time()
# making roads
numRoads = 100
# make a heatmap that shows the distance to the nearest city tile
# have a pathfinding thing pick two random cities and connect them via a walk
# remove any that are on water

# set a map to 100
cost = np.ones((N,N))
cost *= 100

# subtract from it the population of the city
cost = cost - cityMap

# blur it
cost = gaussian_filter(cost, sigma=4)
cost = normalize_map(cost)
cost *= 100

#add the mountains
for y in range(len(heightMap)):
    for x in range(len(heightMap)):
        if not heightMap[y][x] <= seaLevel:
            cost[y][x] += heightMap[y][x]*100

# subtract the ocean
for y in range(len(cost)):
    for x in range(len(cost)):
        if heightMap[y][x] <= seaLevel:
            cost[y][x] = 0

# convert floats to ints
cost = cost.astype(int)

#add some randomness
for y in range(len(cost)):
    for x in range(len(cost)):
        if cost[y][x] > 0:
            cost[y][x] += np.random.randint(0,50)

#add a penalty for certain biomes
for y in range(len(cost)):
    for x in range(len(cost)):
        if biomeMap[y][x] in [0, 1, 2, 3, 4, 5, 6, 7, 13, 14, 15, 22, 26, 25, 21]:
            cost[y][x] *= 5

graph = tcod.path.CustomGraph((N, N))
CARDINAL = [
    [141, 100, 141],
    [100, 0, 100],
    [141, 100, 141],
]

roadMap = np.zeros((N, N))

#add from random points on the edge to make the roads not be self contained to the map
#we want at least 2
numBorderRoads = mapSize - 3
for i in range(numBorderRoads):
    #pick a cardinal direction
    border = np.random.randint(0,4)
    if border == 0:
        cityList.append([0, np.random.randint(0,N)])
    elif border == 1:
        cityList.append([N - 1, np.random.randint(0, N)])
    elif border == 2:
        cityList.append([np.random.randint(0, N), 0])
    elif border == 3:
        cityList.append([np.random.randint(0, N), N-1])

# randomize the city list
#cityList = cityList * 2
np.random.shuffle(cityList)
startCity = cityList[0]

from scipy.spatial import distance
def closest_node_loop(node, nodes, n):
    for i in range(n):
        closest_index = distance.cdist([node], nodes).argmin()
        if i < n-1:
            del nodes[closest_index]
    return nodes[closest_index]

cityIndexes = np.random.randint(0, len(cityList), 2)

result = []
for i in cityList:
    if i not in result:
        result.append(i)

cityList = result.copy()

#connect each city to its nth closest neighbor
for n in range(1,4):
    for j in range(len(cityList)):
        startCity = cityList[j]
        #cityIndexes = np.random.randint(0, len(cityList), 2)
        #remove the city from the cityList
        cityListSubset = cityList.copy()
        del cityListSubset[j]
        endCity = closest_node_loop(startCity, cityListSubset,n)
        # get the path
        graph.add_edges(edge_map=CARDINAL, cost=cost)
        pf = tcod.path.Pathfinder(graph)
        pf.add_root((startCity[0], startCity[1]))
        road = pf.path_to((endCity[0], endCity[1]))
        #make the road map
        for i in road:
            roadMap[i[0]][i[1]] += 1
            #adjust the cost based on the roadMap so the roads converge
            cost[i[0]][i[1]] /= 3
        #startCity = endCity

dt = round(time.time() - st, 1)
print("generated roadMap\t\t{} sec".format(dt))

np.savetxt("roadmap_m3.csv", roadMap, delimiter=",")

print("done")


import logging

from django.shortcuts import render
from django import forms
from django.core.mail import send_mail
from django.http import HttpResponseRedirect
from django.core.urlresolvers import reverse
from django.views.generic.edit import FormView
from django.contrib import messages
from django.utils.translation import ugettext as _

from crispy_forms.helper import FormHelper
from crispy_forms.layout import Submit

from studentsdb.settings import ADMIN_EMAIL

class ContactForm(forms.Form):
    def __init__(self, *args, **kwards):
        # call original initializator
        super(ContactForm, self).__init__(*args, **kwards)

        # this helper object allows us to customize form
        self.helper = FormHelper()

        # form tag attributes
        # self.helper.form_class = 'form-horizontal'
        self.form_method = 'post'
        self.helper.form_tag = False
        self.form_action = reverse('contact_admin')

        # twitter bootstrap styles
        self.helper.help_text_inline = True
        self.helper.html5_required = True
        self.helper.label_class = 'col-sm-2 control-label'
        self.helper.field_class = 'col-sm-10'

        # form buttons
        self.helper.add_input(Submit('send_button', _(u'Send')))

    from_email = forms.EmailField(
        label=_(u"Your Email Address")
    )

    subject = forms.CharField(
        label=_(u"Email Subject"),
        max_length=128
    )
    message = forms.CharField(
        label=_(u"Email Body"),
        max_length=2560,
        widget=forms.Textarea
    )

def contact_admin(request):
    # check it form was posted
    if request.method == 'POST':
        # create a form instance and populate it with data from the request:
        form = ContactForm(request.POST)

        # check whether user data is valid:
        if form.is_valid():
            # send email
            subject = form.cleaned_data['subject']
            message = form.cleaned_data['message']
            from_email = form.cleaned_data['from_email']

            try:
                send_mail(subject, message, from_email, [ADMIN_EMAIL])
            except Exception:
                message = _(u"An error occured during email transfer. Please, "
                            u"try again later.")
                messages.warning(request, message)
                logger = logging.getLogger(__name__)
                logger.exception(message)
            else:
                message = _(u'Message sent successfully!')
                messages.success(request, message)

            # redirect to same contact page with success message
            return HttpResponseRedirect(reverse('contact_admin'))

    # if there was not POST render blank form
    else:
        form = ContactForm()

    return render(request, 'contact_admin/form.html', {'form': form})

class ContactView(FormView):
    template_name = 'contact_admin/form.html'
    form_class = ContactForm
    # success_url = '/email-sent/'
    success_url = '/contact_admin/'

    def get_success_url(self):
        return reverse('contact_admin')

    def form_valid(self, form):
        """This method is called for valid data"""
        subject = form.cleaned_data['subject']
        message = form.cleaned_data['message']
        from_email = form.cleaned_data['from_email']

        try:
            send_mail(subject, message, from_email, [ADMIN_EMAIL])
        except Exception:
            message = _(u"An error occured during email transfer. Please, "
                            u"try again later.")
            messages.warning(self.request, message)
            logger = logging.getLogger(__name__)
            logger.exception(message)
        else:
            message = _(u'Message sent successfully!')
            messages.success(self.request, message)

        return super(ContactView, self).form_valid(form)


class IntCodeProgram():
    def __init__(self, name, program):
        self.program = program
        self.name = name
        self.output = []
        self.switch = {1: self.opt1, 2: self.opt2, 3: self.opt3, 4: self.opt4, 5: self.opt5, 6: self.opt6, 7: self.opt7,
                       8: self.opt8, 9: self.opt9, 99: self.opt99}
        self.inputp = 0
        self.halt = False
        self.relative_base = 0
        self.memory = {}
        self.retpt = 0
        self.ball_last_pos = None
        self.ball_pos = None
        self.bar_x = 0
        self.projected_ball_x = 0
        self.score = 0

    def run(self, inputs):
        if not self.halt:
            pp = self.retpt
            while pp >= 0:
                # console.log("executing pp {}".format(pp))
                opt_code = program[pp] % 100
                mode = str(program[pp])[:-2].zfill(3)
                pp = self.switch[opt_code](self.program, pp, mode, inputs)
        print("Done. Score: {}".format(self.score))
        return self.output

    def read_value(self, pp, pp_offset, mode):
        index = 0
        if mode == '0':
            index = self.program[pp + pp_offset]
        if mode == '1':
            index = pp + pp_offset
        if mode == '2':
            index = self.relative_base + self.program[pp + pp_offset]
        if index >= len(self.program):
            return self.memory.get(index, 0)
        return self.program[index]

    def write_to_memory(self, index, value, mode):
        if mode == '2':
            index = self.relative_base + index
        if index >= len(self.program):
            self.memory[index] = value
        else:
            self.program[index] = value

    def update_bar_position(self):
        self.bar_x = self.output[-3]

    def update_ball_position(self):
        if self.ball_last_pos:
            self.ball_last_pos = self.ball_pos
        self.ball_pos = (self.output[-3], self.output[-2])
        if not self.ball_last_pos:
            self.ball_last_pos = (self.output[-3], self.output[-2])
        self.projected_ball_x = self.ball_pos[0]

    def opt1(self, prog, pp, opt_mode, _):
        m1 = opt_mode[2]
        m2 = opt_mode[1]
        a = self.read_value(pp, 1, m1)
        b = self.read_value(pp, 2, m2)
        self.write_to_memory(prog[pp + 3], a + b, opt_mode[0])
        return pp + 4

    def opt2(self, prog, pp, mode, _):
        m1 = mode[2]
        m2 = mode[1]
        a = self.read_value(pp, 1, m1)
        b = self.read_value(pp, 2, m2)
        self.write_to_memory(prog[pp + 3], a * b, mode[0])
        return pp + 4

    def opt3(self, prog, pp, mode, inputs):
        inp = 0
        if self.projected_ball_x < self.bar_x:
            inp = -1
        elif self.projected_ball_x > self.bar_x:
            inp = 1
        self.write_to_memory(prog[pp + 1], inp, mode[2])
        return pp + 2

    def opt4(self, prog, pp, mode, _):
        out = self.read_value(pp, 1, mode[2])
        self.output.append(out)
        if len(self.output) % 3 == 0 and self.output[-3] >= 0:
            # print("updating position: {}".format(self.output[-3:]))
            if self.output[-1] == 4:
                self.update_ball_position()

            elif self.output[-1] == 3:
                self.update_bar_position()
        elif len(self.output) % 3 == 0 and self.output[-3] == -1:
            self.score = self.output[-1]
        self.retpt = pp + 2
        return pp + 2

    def opt5(self, prog, pp, mode, _):
        m1 = mode[2]
        m2 = mode[1]
        x = self.read_value(pp, 1, m1)
        if x != 0:
            return self.read_value(pp, 2, m2)
        return pp + 3

    def opt6(self, prog, pp, mode, _):
        m1 = mode[2]
        m2 = mode[1]
        x = self.read_value(pp, 1, m1)
        if x == 0:
            return self.read_value(pp, 2, m2)
        return pp + 3

    def opt7(self, prog, pp, mode, _):
        m1 = mode[2]
        m2 = mode[1]
        x = self.read_value(pp, 1, m1)
        y = self.read_value(pp, 2, m2)
        ret = 0
        if x < y:
            ret = 1
        self.write_to_memory(prog[pp + 3], ret, mode[0])
        return pp + 4

    def opt8(self, prog, pp, mode, _):
        m1 = mode[2]
        m2 = mode[1]
        m3 = mode[0]
        x = self.read_value(pp, 1, m1)
        y = self.read_value(pp, 2, m2)
        ret = 0
        if x == y:
            ret = 1
        self.write_to_memory(prog[pp + 3], ret, m3)
        return pp + 4

    def opt9(self, prog, pp, mode, _2):
        inc = self.read_value(pp, 1, mode[2])
        self.relative_base += inc
        return pp + 2

    def opt99(self, prog, _1, _2, _3):
        self.halt = True
        return -1

with open('input') as f:
    program = list(map(lambda x: int(x), f.readline().split(',')))

drawer = IntCodeProgram('draw', program)
drawer.run([])


# Import required libraries
import matplotlib.pyplot as plt
import cv2
import numpy as np
import os

# Read source and mask (if exists) for a given id
def Read(id, path = ""):
    source = plt.imread(path + "image_" + id + ".jpg")
    maskPath = path + "mask_" + id + ".jpg"

    if os.path.isfile(maskPath):
        mask = plt.imread(maskPath)
        assert(mask.shape == source.shape), 'size of mask and image does not match'
        mask = (mask > 128)[:, :, 0].astype(int)
    else:
        mask = np.zeros_like(source)[:, :, 0].astype(int)

    return source, mask

def compute_energy_matrix(img):
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

    dx = cv2.Sobel(gray, cv2.CV_64F, 1, 0)
    dy = cv2.Sobel(gray, cv2.CV_64F, 0, 1)
    energy = np.abs(dx) + np.abs(dy)

    return energy

def compute_optimal_seam(E):
    # Make the scoring matrix M
    M = E.copy()

    # Set the values of every entry in the matrix except for the first row
    for i in range(1, M.shape[0]):
        for j in range(M.shape[1]):
            # Need to compensate for bounds
            if j == 0:
                M[i][j] = E[i][j] + min(M[i - 1][j], M[i - 1][j + 1])
            elif j == M.shape[1] - 1:
                M[i][j] = E[i][j] + min(M[i - 1][j - 1], M[i - 1][j])
            else:
                M[i][j] = E[i][j] + min(M[i - 1][j - 1], M[i - 1][j], M[i - 1][j + 1])
    return M

def find_min_value(M):
    to_list = list(M)
    return to_list.index(min(to_list))

def compute_seam_path(M):
    h, w = M.shape
    min_val_bot_rpw = find_min_value(M[h - 1, :])
    seam = [min_val_bot_rpw]
    i = 0

    for y in range(h - 1, 0, -1):
        x = seam[i]

        if (x == 0):
            min_val = find_min_value(M[y - 1, x : x + 2])
            seam.append(x + min_val)
        elif(x == w - 1):
            min_val = find_min_value(M[y - 1, x - 1: x + 1])
            seam.append(x + min_val - 1)
        else:
            min_val = find_min_value(M[y - 1, x - 1: x + 2])
            seam.append(x + min_val - 1)
        i += 1

    return seam

def seam_removal(img, seam):
    h, w, _ = img.shape
    output = cv2.resize(img, (w - 1, h), interpolation=cv2.INTER_LINEAR)
    col = h - 1

    for i in range(img.shape[0]):
        row = seam[i]
        output[col, : row] = img[col, : row]
        output[col, row:] = img[col, row + 1:]
        col = col - 1

    return output

def scale_column(input, scale_c):
    target_width = int(input.shape[1] * scale_c)

    for i in range(input.shape[1] - target_width):
        E = compute_energy_matrix(input)
        M = compute_optimal_seam(E)
        seam = compute_seam_path(M)
        input = seam_removal(input, seam)

    return input

def scale_row(input, scale_r):
    input = np.rot90(input, 1)
    input = scale_column(input, scale_r)

    return np.rot90(input, 3)

def SeamCarve(input, widthFac, heightFac, mask):

    # Main seam carving function. This is done in three main parts: 1)
    # computing the energy function, 2) finding optimal seam, and 3) removing
    # the seam. The three parts are repeated until the desired size is reached.

    assert (widthFac == 1 or heightFac == 1), 'Changing both width and height is not supported!'
    assert (widthFac <= 1 and heightFac <= 1), 'Increasing the size is not supported!'

    if heightFac != 1:
        input = scale_row(input, heightFac)
    else:
        input = scale_column(input, widthFac)

    return input, (input.shape[1], input.shape[0])

# Setting up the input output paths
inputDir = '../Images/'
outputDir = '../Results/'

widthFac = 1; # To reduce the width, set this parameter to a value less than 1
heightFac = 0.5;  # To reduce the height, set this parameter to a value less than 1
N = 4 # number of images

for index in range(N, N + 1):

    input, mask = Read(str(index).zfill(2), inputDir)

    # Performing seam carving. This is the part that you have to implement.
    output, size = SeamCarve(input, widthFac, heightFac, mask)

    # Writing the result
    plt.imsave("{}/result_{}_{}x{}.jpg".format(outputDir,
                                            str(index).zfill(2),
                                            str(size[0]).zfill(2),
                                            str(size[1]).zfill(2)), output)


import sys
import random
from constants import N_OPTION
from print import o

from sym import Sym
from utils import per

class Num(Sym):
  def __init__(self, c = 0, s = ""):
    super().__init__(c, s)
    self.n = 0
    self.at = c or 0
    self.name = s or ""
    self._has = []
    self.lo = sys.maxsize
    self.hi = -sys.maxsize
    self.isSorted = True
    self.w = -1 if (s or "").find("$") == -1 else 1

  def __repr__(self):
    return o({"at": self.at, "hi": self.hi, "isSorted": self.isSorted, "lo": self.lo, "n": self.n,
    "name": self.name, "w": self.w})

  def nums(self):
    if not self.isSorted:
      self._has.sort()
      self.isSorted = True
    return self._has

  # v is a number -> value like key, value
  def add(self, v, nums, rand: random.Random):
    if v!="?":
      self.n = self.n +1
      self.lo = min(v, self.lo)
      self.hi = max(v, self.hi)

      if len(self._has) < nums:
        self.isSorted = False
        self._has.append(float(v))

      elif rand.random() < nums/self.n:
        # generate random int b/w 1, len of _has. As mentioned in csv.luna
        self.isSorted = False
        pos = rand.randint(0, len(self._has)-1)
        self._has[pos] = float(v)

  def div(self):
    a = self.nums()
    return (per(a,.9) - per(a,0.1))/2.58

  def mid(self):
    return per(self.nums(),.5)


import collections

import cv2
import gym
import gym_super_mario_bros
import numpy as np
import torch
from gym.core import Env
from nes_py.wrappers import JoypadSpace

from src.constants import (
    AGENT_ACTIONS,
    ENV_HEIGHT,
    ENV_WIDTH,
    SKIP_AND_STACK_AMOUNT,
    WORLD,
)

class MaxAndSkipEnv(gym.Wrapper):
    def __init__(self, env=None, skip=SKIP_AND_STACK_AMOUNT):
        """Return only every `skip`-th frame"""
        super(MaxAndSkipEnv, self).__init__(env)
        # most recent raw observations (for max pooling across time steps)
        self._obs_buffer = collections.deque(maxlen=2)
        self._skip = skip

    def step(self, action):
        total_reward = 0.0
        done = None
        for _ in range(self._skip):
            obs, reward, done, info = self.env.step(action)
            self._obs_buffer.append(obs)
            total_reward += reward
            if done:
                break
        max_frame = np.max(np.stack(self._obs_buffer), axis=0)
        # Save the original state to allow fetching when creating video
        self.frame = max_frame.copy()
        return max_frame, total_reward, done, info

    def reset(self):
        """Clear past frame buffer and init to first obs"""
        self._obs_buffer.clear()
        obs = self.env.reset()
        self._obs_buffer.append(obs)
        return obs

class ProcessFrame84(gym.ObservationWrapper):
    """
    Downsamples image to 84x84
    Greyscales image

    Returns numpy array
    """

    def __init__(self, env=None):
        super(ProcessFrame84, self).__init__(env)
        self.observation_space = gym.spaces.Box(
            low=0, high=255, shape=(ENV_WIDTH, ENV_HEIGHT, 1), dtype=np.uint8
        )

    def observation(self, obs):
        return ProcessFrame84.process(obs)

    @staticmethod
    def process(frame):
        if frame.size == 240 * 256 * 3:
            img = np.reshape(frame, [240, 256, 3]).astype(np.float32)
        else:
            assert False, "Unknown resolution."
        img = img[:, :, 0] * 0.299 + img[:, :, 1] * 0.587 + img[:, :, 2] * 0.114
        resized_screen = cv2.resize(img, (84, 110), interpolation=cv2.INTER_AREA)
        x_t = resized_screen[18:102, :]
        x_t = np.reshape(x_t, [84, 84, 1])
        return x_t.astype(np.uint8)

class ImageToPyTorch(gym.ObservationWrapper):
    """Class to convert a state image to a pytorch tensor
    """

    def __init__(self, env):
        super(ImageToPyTorch, self).__init__(env)
        old_shape = self.observation_space.shape
        self.observation_space = gym.spaces.Box(
            low=0.0,
            high=1.0,
            shape=(old_shape[-1], old_shape[0], old_shape[1]),
            dtype=np.float32,
        )

    def observation(self, observation):
        return np.moveaxis(observation, 2, 0)

class ScaledFloatFrame(gym.ObservationWrapper):
    """Normalize pixel values in frame --> 0 to 1"""

    def observation(self, obs):
        return np.array(obs).astype(np.float32) / 255.0

class BufferWrapper(gym.ObservationWrapper):
    """Class to wrapper observation in a buffer"""

    def __init__(self, env, n_steps, dtype=np.float32):
        super(BufferWrapper, self).__init__(env)
        self.dtype = dtype
        old_space = env.observation_space
        self.observation_space = gym.spaces.Box(
            old_space.low.repeat(n_steps, axis=0),
            old_space.high.repeat(n_steps, axis=0),
            dtype=dtype,
        )

    def reset(self):
        self.buffer = np.zeros_like(self.observation_space.low, dtype=self.dtype)
        return self.observation(self.env.reset())

    def observation(self, observation):
        self.buffer[:-1] = self.buffer[1:]
        self.buffer[-1] = observation
        return self.buffer

def create_mario_env() -> Env:
    """Function to create a Super Mario Bros environment wrapped by our custom  wrappers

    Returns:
        Env: the wrapped environment
    """
    env = gym_super_mario_bros.make(WORLD)
    env = MaxAndSkipEnv(env)
    env = ProcessFrame84(env)
    env = ImageToPyTorch(env)
    env = BufferWrapper(env, 4)
    env = ScaledFloatFrame(env)
    env.seed(42)
    env.action_space.seed(42)
    torch.manual_seed(42)
    torch.random.manual_seed(42)
    np.random.seed(42)
    return JoypadSpace(env, AGENT_ACTIONS)

IDATT2502 Anvendt maskinlæring med prosjekt/project/src/environment.py


import torch
import torch.nn as nn
import torch.nn.functional as F
from sparsebit.sparse.modules import SparseOpr, register_smodule

@register_smodule(sources=[nn.Conv2d])
class SConv2d(SparseOpr):
    def __init__(self, org_module, config=None):
        assert isinstance(org_module, nn.Conv2d)
        super().__init__()
        self.fwd_kwargs = dict(
            stride=org_module.stride,
            padding=org_module.padding,
            dilation=org_module.dilation,
            groups=org_module.groups,
        )
        self.weight = org_module.weight
        self.bias = org_module.bias
        w_mask = torch.ones_like(self.weight)
        b_mask = torch.ones_like(self.bias) if self.bias is not None else None
        self.register_buffer("w_mask", w_mask)
        self.register_buffer("b_mask", b_mask)
        self._repr_info = "S" + org_module.__repr__()

    def calc_mask(self, pre_mask=None):
        self.w_mask = self.sparser.calc_mask(self.weight)

        if self.sparser.type == "structed":
            mask = self.w_mask[:, 0, 0, 0]
            if self.bias is not None:
                self.b_mask.data.copy_(mask.data)

            if self.sparser.strategy == "l1norm":
                return mask

        return None

    def forward(self, x_in: torch.Tensor):
        weight = self.weight * self.w_mask
        bias = self.bias * self.b_mask if self.bias is not None else self.bias
        out = F.conv2d(x_in, weight, bias, **self.fwd_kwargs)
        return out


import numpy as np
import pytest
from numpy.testing import assert_allclose, assert_array_equal
from scipy.optimize import minimize
from sklearn.isotonic import IsotonicRegression as skl_IsotonicRegression

from model_diagnostics._utils.isotonic import (
    IsotonicRegression,
    gpava,
    isotonic_regression,
    pava,
)
from model_diagnostics.scoring import PinballLoss, SquaredError

def mean_fun(x, w):
    return np.average(x, weights=w)

mean_fun.loss = SquaredError()  # type: ignore

def median_lower(x, w=None):
    return np.quantile(x, q=0.5, method="inverted_cdf")

median_lower.loss = PinballLoss(level=0.5)  # type: ignore

def median_upper(x, w=None):
    return -np.quantile(-np.asarray(x), q=0.5, method="inverted_cdf")

median_upper.loss = PinballLoss(level=0.5)  # type: ignore

def quantile80_lower(x, w=None):
    return np.quantile(x, q=0.8, method="inverted_cdf")

quantile80_lower.loss = PinballLoss(level=0.8)  # type: ignore

def quantile80_upper(x, w=None):
    return -np.quantile(-np.asarray(x), q=1 - 0.8, method="inverted_cdf")

quantile80_upper.loss = PinballLoss(level=0.8)  # type: ignore

def quantile10_lower(x, w=None):
    return np.quantile(x, q=0.1, method="inverted_cdf")

quantile10_lower.loss = PinballLoss(level=0.1)  # type: ignore

def quantile10_upper(x, w=None):
    return -np.quantile(-np.asarray(x), q=1 - 0.1, method="inverted_cdf")

quantile10_upper.loss = PinballLoss(level=0.1)  # type: ignore

def gpava_mean(x, w=None):
    return gpava(mean_fun, x, w)

@pytest.mark.parametrize("pava", [pava, gpava_mean])
def test_pava_simple(pava):
    # Test case of Busing 2020
    # https://doi.org/10.18637/jss.v102.c01
    y = [8, 4, 8, 2, 2, 0, 8]
    w = np.ones_like(y)
    x, r = pava(y, w)
    assert_array_equal(x, [4, 4, 4, 4, 4, 4, 8])
    assert_array_equal(r, [0, 6, 7])

    y = [9, 1, 8, 2, 7, 3, 6]
    w = None
    x, r = pava(y, w)
    assert_array_equal(x, [5, 5, 5, 5, 5, 5, 6])
    assert_array_equal(r, [0, 6, 7])

    y = [9, 1, 8, 4, -2, 8, 6]
    x, r = pava(y, w)
    assert_array_equal(x, [4, 4, 4, 4, 4, 7, 7])
    assert_array_equal(r, [0, 5, 7])

@pytest.mark.parametrize("pava", [pava, gpava_mean])
def test_pava_weighted(pava):
    y = [8, 4, 8, 2, 2, 0, 8]
    w = [1, 2, 3, 4, 5, 5, 4]
    x, r = pava(y, w)
    assert_array_equal(x, [2.9, 2.9, 2.9, 2.9, 2.9, 2.9, 8])
    assert_array_equal(r, [0, 6, 7])

@pytest.mark.parametrize(
    ("y", "fun_lower", "x_lower", "fun_upper", "x_upper"),
    [
        (
            [8, 4, 8, 2, 2, 0, 8],
            median_lower,
            [2, 2, 2, 2, 2, 2, 8],
            median_upper,
            [4, 4, 4, 4, 4, 4, 8],
        ),
        (
            [9, 1, 8, 2, 7, 3, 6],
            median_lower,
            [1, 1, 2, 2, 3, 3, 6],
            median_upper,
            [6, 6, 6, 6, 6, 6, 6],
        ),
        (
            [9, 1, 8, 4, -2, 8, 6],
            median_lower,
            [1, 1, 4, 4, 4, 6, 6],
            median_upper,
            [4, 4, 4, 4, 4, 8, 8],
        ),
        (
            [8, 4, 8, 2, 2, 0, 8],
            quantile80_lower,
            [8, 8, 8, 8, 8, 8, 8],
            quantile80_upper,
            [8, 8, 8, 8, 8, 8, 8],
        ),
        (
            [9, 1, 8, 2, 7, 3, 6],
            quantile80_lower,
            [8, 8, 8, 8, 8, 8, 8],
            quantile80_upper,
            [8, 8, 8, 8, 8, 8, 8],
        ),
        (
            [9, 1, 8, 4, -2, 8, 6],
            quantile80_lower,
            [8, 8, 8, 8, 8, 8, 8],
            quantile80_upper,
            [8, 8, 8, 8, 8, 8, 8],
        ),
        (
            [8, 4, 4, 4, 0, 8, 2, 2, 2, 0, 8],
            median_lower,
            [2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 8],
            median_upper,
            [4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 8],
        ),
        (
            [
                -3.444949308,
                -2.854784791,
                -3.071818855,
                -1.164616209,
                -3.947223934,
                -2.644162705,
                -2.609553073,
                -0.270605801,
                -3.208354459,
                -2.012322283,
                -1.059124491,
                -2.199654113,
                -1.906487545,
                -2.681030502,
                -1.553997732,
                -3.489485619,
                -1.672190775,
                -2.863511214,
                -0.491263988,
                0.169577379,
            ],
            quantile10_lower,
            # from R package isotonic, function
            # gpava(1:20, y, solver = weighted.fractile, p=0.1, ties = "secondary")$x
            np.repeat(
                [-3.947223934, -3.208354459, -2.863511214, -0.491263988, 0.169577379],
                [5, 11, 2, 1, 1],
            ),
            quantile10_upper,
            np.repeat(
                [-3.947223934, -3.208354459, -2.863511214, -0.491263988, 0.169577379],
                [5, 11, 2, 1, 1],
            ),
        ),
    ],
)
def test_gpava_simple(y, fun_lower, x_lower, fun_upper, x_upper):
    w = np.ones_like(y)
    xl, rl = gpava(fun_lower, y, w)
    # The distinction of lower and upper only applies to set-valued functionals like
    # quantiles.
    # Applying gpava on fun=np.quantile(x, level, method="higher") does not work
    # for unknown reasons. What works is to calculate the upper quantile on the
    # block given by rl.
    upper_values = np.fromiter(
        (fun_upper(y[rl[i] : rl[i + 1]]) for i in range(len(rl) - 1)), dtype=xl.dtype
    )
    # Take mininum from the right.
    upper_values = np.minimum.accumulate(upper_values[::-1])[::-1]
    xu = np.repeat(upper_values, np.diff(rl))

    # Check that isotonic_regression gives midpoint.
    functional = fun_lower.loss.functional
    level = getattr(fun_lower.loss, "level", 0.5)
    x_iso, _ = isotonic_regression(y=y, functional=functional, level=level)
    assert_allclose(0.5 * (xl + xu), x_iso)

    for i in range(len(xl) - 1):
        # Eq. 10 of Jordan et al. https://doi.org/10.1007/s10463-021-00808-0
        assert xl[i] <= xu[i + 1]

    # Check manually precomputed values.
    assert_array_equal(xl, x_lower)
    assert_array_equal(xu, x_upper)

    # Check that it is a minimum.
    loss_lower = fun_lower.loss(y_obs=y, y_pred=xl)
    loss_upper = fun_lower.loss(y_obs=y, y_pred=xu)
    assert loss_lower == pytest.approx(loss_upper)

    def objective(y_pred, y_obs):
        return fun_lower.loss(y_obs=y_obs, y_pred=y_pred)

    def constraint(y_pred, y_obs):
        # This is for a monotonically increasing regression.
        return np.diff(y_pred)

    result = minimize(
        fun=objective,
        x0=y,
        args=(y,),
        constraints=[{"type": "ineq", "fun": lambda x: constraint(x, y)}],
        tol=1e-12,
    )

    assert result.fun == pytest.approx(loss_lower, rel=1e-8)

    # Check that it is the boundary of solutions.
    eps = 1e-8 * np.mean(np.abs(xl))
    for i in range(len(xl)):
        x = np.array(xl, copy=True, dtype=float)
        if i == 0 or x[i] > x[i - 1]:
            x[i] -= eps
            assert fun_lower.loss(y_obs=y, y_pred=x) > loss_lower

        x = np.array(xu, copy=True, dtype=float)
        if i == len(xl) - 1 or x[i] < x[i + 1]:
            x[i] += eps
            assert fun_upper.loss(y_obs=y, y_pred=x) > loss_upper

@pytest.mark.parametrize(
    ("functional", "level", "msg"),
    [
        ("no good functional", 0.5, "Argument functional must be one of"),
        ("quantile", 0, "Argument level must fulfil 0 < level < 1"),
        ("quantile", 1, "Argument level must fulfil 0 < level < 1"),
        ("quantile", 1.1, "Argument level must fulfil 0 < level < 1"),
    ],
)
def test_isotonic_regression_raises(functional, level, msg):
    y, w = np.arange(5), np.arange(5)
    with pytest.raises(ValueError, match=msg):
        isotonic_regression(y=y, weights=w, functional=functional, level=level)

@pytest.mark.parametrize("functional", ["quantile", "median"])
def test_isotonic_regression_raises_weighted_quantile(functional):
    y, w = np.arange(5), np.arange(5)
    msg = "Weighted quantile"
    with pytest.raises(NotImplementedError, match=msg):
        isotonic_regression(y=y, weights=w, functional=functional, level=0.5)

@pytest.mark.parametrize(
    ("weights", "msg"),
    [
        ([0, 1, 2], "Input arrays y and w must have one dimension of equal length."),
        ([0, -1], "Weights w must be strictly positive."),
    ],
)
def test_isotonic_regression_raises_weight_array_validation(weights, msg):
    """Test input validation of weight array."""
    y = [0, 1]
    with pytest.raises(ValueError, match=msg):
        isotonic_regression(y=y, weights=weights)

@pytest.mark.parametrize("w", [None, np.ones(7)])
def test_simple_isotonic_regression(w):
    # Test case of Busing 2020
    # https://doi.org/10.18637/jss.v102.c01
    y = np.array([8, 4, 8, 2, 2, 0, 8], dtype=np.float64)
    x, r = isotonic_regression(y, w)
    assert_allclose(x, [4, 4, 4, 4, 4, 4, 8])
    assert_allclose(np.add.reduceat(np.ones_like(y), r[:-1]), [6, 1])
    assert_allclose(r, [0, 6, 7])
    # Assert that y was not overwritten
    assert_array_equal(y, np.array([8, 4, 8, 2, 2, 0, 8], dtype=np.float64))

@pytest.mark.parametrize(
    ("functional", "level", "x_res", "r_res"),
    [
        ("mean", None, [4, 4, 4, 4, 4, 4, 8], [0, 6, 7]),
        ("median", None, [3, 3, 3, 3, 3, 3, 8], [0, 6, 7]),
        ("quantile", 0.5, [3, 3, 3, 3, 3, 3, 8], [0, 6, 7]),
        ("quantile", 0.8, [8, 8, 8, 8, 8, 8, 8], [0, 7]),
        ("expectile", 0.5, [4, 4, 4, 4, 4, 4, 8], [0, 6, 7]),
        ("expectile", 0.8, [6, 6, 6, 6, 6, 6, 8], [0, 6, 7]),
    ],
)
def test_simple_isotonic_regression_functionals(functional, level, x_res, r_res):
    # Test case of Busing 2020
    # https://doi.org/10.18637/jss.v102.c01
    y = np.array([8, 4, 8, 2, 2, 0, 8], dtype=np.float64)
    x, r = isotonic_regression(y, functional=functional, level=level)
    assert_allclose(x, x_res)
    assert_allclose(r, r_res)
    # Assert that y was not overwritten
    assert_array_equal(y, np.array([8, 4, 8, 2, 2, 0, 8], dtype=np.float64))

@pytest.mark.parametrize("increasing", [True, False])
def test_linspace(increasing):
    n = 10
    y = np.linspace(0, 1, n) if increasing else np.linspace(1, 0, n)
    x, r = isotonic_regression(y, increasing=increasing)
    assert_allclose(x, y)
    assert_array_equal(r, np.arange(n + 1))

def test_weights():
    w = np.array([1, 2, 5, 0.5, 0.5, 0.5, 1, 3])
    y = np.array([3, 2, 1, 10, 9, 8, 20, 10])
    x, r = isotonic_regression(y, w)
    assert_allclose(x, [12 / 8, 12 / 8, 12 / 8, 9, 9, 9, 50 / 4, 50 / 4])
    assert_allclose(np.add.reduceat(w, r[:-1]), [8, 1.5, 4])
    assert_array_equal(r, [0, 3, 6, 8])

    # weights are like repeated observations, we repeat the 3rd element 5
    # times.
    w2 = np.array([1, 2, 1, 1, 1, 1, 1, 0.5, 0.5, 0.5, 1, 3])
    y2 = np.array([3, 2, 1, 1, 1, 1, 1, 10, 9, 8, 20, 10])
    x2, r2 = isotonic_regression(y2, w2)
    assert_allclose(np.diff(x2[0:7]), 0)
    assert_allclose(x2[4:], x)
    assert_allclose(np.add.reduceat(w2, r2[:-1]), np.add.reduceat(w, r[:-1]))
    assert_array_equal(r2 - [0, 4, 4, 4], r)

def test_against_R_monotone():
    y = [0, 6, 8, 3, 5, 2, 1, 7, 9, 4]
    x, r = isotonic_regression(y)
    # R code
    # library(monotone)
    # options(digits=8)
    # monotone(c(0, 6, 8, 3, 5, 2, 1, 7, 9, 4))
    res = [
        0,
        4.1666667,
        4.1666667,
        4.1666667,
        4.1666667,
        4.1666667,
        4.1666667,
        6.6666667,
        6.6666667,
        6.6666667,
    ]
    assert_allclose(x, res)
    assert_array_equal(r, [0, 1, 7, 10])

    n = 100
    y = np.linspace(0, 1, num=n, endpoint=False)
    y = 5 * y + np.sin(10 * y)
    x, r = isotonic_regression(y)
    # R code
    # library(monotone)
    # y <- 5 * ((1:n)-1)/n + sin(10 * ((1:n)-1)/n)
    # monotone(y)
    res = [
        0.0000000,
        0.1498334,
        0.2986693,
        0.4455202,
        0.5894183,
        0.7294255,
        0.8646425,
        0.9942177,
        1.1173561,
        1.2333269,
        1.3414710,
        1.4412074,
        1.5320391,
        1.5708110,
        1.5708110,
        1.5708110,
        1.5708110,
        1.5708110,
        1.5708110,
        1.5708110,
        1.5708110,
        1.5708110,
        1.5708110,
        1.5708110,
        1.5708110,
        1.5708110,
        1.5708110,
        1.5708110,
        1.5708110,
        1.5708110,
        1.5708110,
        1.5708110,
        1.5708110,
        1.5708110,
        1.5708110,
        1.5708110,
        1.5708110,
        1.5708110,
        1.5708110,
        1.5708110,
        1.5708110,
        1.5708110,
        1.5708110,
        1.5708110,
        1.5708110,
        1.5708110,
        1.5708110,
        1.5708110,
        1.5708110,
        1.5708110,
        1.5708110,
        1.6241853,
        1.7165453,
        1.8177326,
        1.9272355,
        2.0444597,
        2.1687334,
        2.2993145,
        2.4353978,
        2.5761233,
        2.7205845,
        2.8678375,
        3.0169106,
        3.1668139,
        3.3165492,
        3.4651200,
        3.6115414,
        3.7548499,
        3.8941134,
        4.0284398,
        4.1569866,
        4.2789690,
        4.3936679,
        4.5004366,
        4.5987081,
        4.6880000,
        4.7679197,
        4.8381682,
        4.8656413,
        4.8656413,
        4.8656413,
        4.8656413,
        4.8656413,
        4.8656413,
        4.8656413,
        4.8656413,
        4.8656413,
        4.8656413,
        4.8656413,
        4.8656413,
        4.8656413,
        4.8656413,
        4.8656413,
        4.8656413,
        4.8656413,
        4.8656413,
        4.8656413,
        4.8656413,
        4.8656413,
        4.8656413,
    ]
    assert_allclose(x, res, rtol=2e-7)

    # Test increasing
    assert np.all(np.diff(x) >= 0)

    # Test balance property: sum(y) == sum(x)
    assert_allclose(np.sum(x), np.sum(y))

    # Reverse order
    x, rinv = isotonic_regression(-y, increasing=False)
    assert_allclose(-x, res, rtol=2e-7)
    assert_array_equal(rinv, r)

@pytest.mark.parametrize("increasing", [True, False])
def test_isotonic_regression_class(increasing):
    """Test that IsotonicRegression gives the same as the scikit-learn version."""
    X = [0, 2, 4, -1, -2, 3, 2, 2, 1, 4]
    y = np.arange(10)
    m_skl = skl_IsotonicRegression(increasing=increasing, out_of_bounds="clip").fit(
        X, y
    )
    m = IsotonicRegression(increasing=increasing).fit(X, y)

    assert_allclose(m.X_thresholds_, m_skl.X_thresholds_)
    assert_allclose(m.y_thresholds_, m_skl.y_thresholds_)

    assert m.X_thresholds_[0] == m_skl.X_min_
    assert m.X_thresholds_[-1] == m_skl.X_max_

    X_pred = [-10, -2, 1, 2.5, 10]
    assert_allclose(m.predict(X_pred), m_skl.predict(X_pred))

    m_exp = IsotonicRegression(increasing=increasing, functional="expectile").fit(X, y)
    assert_allclose(m.X_thresholds_, m_exp.X_thresholds_)
    assert_allclose(m.y_thresholds_, m_exp.y_thresholds_)
    assert_allclose(m.predict(X_pred), m_exp.predict(X_pred))

def test_isotonic_regression_class_median():
    """Test IsotonicRegression for median regression."""
    rng = np.random.default_rng(42)
    # Test case of Busing 2020
    # https://doi.org/10.18637/jss.v102.c01
    y = np.array([8, 4, 8, 2, 2, 0, 8], dtype=np.float64)
    y_iso = np.array([3, 3, 3, 3, 3, 3, 8])
    X = np.arange(len(y))
    idx = rng.permutation(X)

    m = IsotonicRegression(functional="quantile", level=0.5).fit(X[idx], y[idx])
    assert_allclose(m.X_thresholds_, [0, 5, 6])
    assert_allclose(m.y_thresholds_, [3, 3, 8])
    assert_allclose(m.predict(X), y_iso)


import os
from tweepy.streaming import StreamListener
from tweepy import OAuthHandler
from tweepy import Stream
from kafka import SimpleProducer, KafkaClient
from dotenv import load_dotenv, find_dotenv

load_dotenv(find_dotenv())

access_token = os.getenv('ACCESS_TOKEN')
access_token_secret = os.getenv('ACCESS_TOKEN_SECRET')
consumer_key = os.getenv('CONSUMER_KEY')
consumer_secret = os.getenv('CONSUMER_SECRET')

class StdOutListener(StreamListener):
    def on_data(self, data):
        producer.send_messages('twitter', data.encode('utf-8'))
        return True

    def on_error(self, status):
        print (status)

kafka = KafkaClient('localhost:9092')
producer = SimpleProducer(kafka)
l = StdOutListener()
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
stream = Stream(auth, l)
stream.filter(languages=["en"], track=['basketball', 'nba', 'ncaa'])


x="一二三四五六qwert12345"
print(len(x))
y=str(1.2345678)
print(y)
a=' '
n=5
for i in range(12):
    print(chr(9800+i)+a*n,end="")
   # print(" ")
print("\n")
z="ASJHBSJCN"
print(str.lower(z))

s1="pythonnpnpnp".strip("np")
print(s1)

s2="{0:=^20}".format("Ptython")
print(s2)

import time
s3=time.ctime()
print(s3)

import turtle
turtle.Pen


""" je ne m'attarde pas sur ce pb queleque petite explication :
cette solution utilise un max heap on prend a chaque fois les 2 max et on retire de chacun 1 . """
class Solution:
    def maximumScore(self, a: int, b: int, c: int) -> int:
            heap = [-a,-b,-c]
            heapq.heapify(heap)   # O(n)  (n taille du heap)
            score =0
            # cette loop est O(2*10^5) car a chaque fois on choisi 2 nombre on enleve 1 et on les remet dans le heap , or le nmb est max 10^5 donc on peut le faire O(2*10^5) si a=b=c=10^5
            while len(heap) >= 2:      # when remaining length >=2
                p1 = heapq.heappop(heap) # O(logn)
                p2 = heapq.heappop(heap) # O(logn)
                if p1+1 !=0:
                    heapq.heappush(heap,p1+1) # O(logn)
                if p2+1 != 0:
                    heapq.heappush(heap,p2+1) # O(logn)
                score += 1
            return score

"""il ya une solution mathematique O(1) : https://leetcode.com/problems/maximum-score-from-removing-stones/discuss/1053645/Python3-math """

1753. Maximum Score From Removing Stones/solution.py


# Write your frequency_dictionary function here:
def frequency_dictionary(words):
  my_storage = {}
  for word in words:
    if word not in my_storage:
      my_storage[word] =+ 0
    my_storage[word] =+ 1
  return my_storage

# Uncomment these function calls to test your  function:
print(frequency_dictionary(["apple", "apple", "cat", 1]))
# should print {"apple":2, "cat":1, 1:1}
print(frequency_dictionary([0,0,0,0,0]))
# should print {0:5}

"""
To create a new dictionary we set a variable equal to {}.
We iterate through each of the strings in the list of strings and check if it is already in our dictionary using the in keyword.
If it is not then we add it as a new key-value pair where the value is 0.
Regardless of whether the string was already in the dictionary, increase the value by 1.
This will make it so all new entries will have a value of 1 and all existing entries will increase their old value by 1.
"""


# Advent of Code 2021
# Day 1

import os

def main():
    with open(f"input{os.sep}day01.txt") as puzzle_data:
        puzzle_input = puzzle_data.read().split()

    previous_depth = None
    total_changes = 0
    for current_depth in puzzle_input:
        if previous_depth is not None:
            if int(current_depth) > int(previous_depth):
                total_changes += 1

        previous_depth = current_depth

    print(f"The number of times the depth increases is {total_changes}.")

    total_sum_changes = 0
    for i in range(len(puzzle_input) - 3):
        if int(puzzle_input[i]) < int(puzzle_input[i+3]):
            total_sum_changes += 1

    print(f"The number of times the moving sum changes is {total_sum_changes}.")

if __name__ == "__main__":
    main()


"""
This module controls the notes for every papis document.
"""
import os

import papis.config
import papis.api
import papis.format
import papis.document
import papis.utils
import papis.hooks
import papis.logging

logger = papis.logging.get_logger(__name__)

def has_notes(doc: papis.document.Document) -> bool:
    """Checks if the document has notes."""
    return "notes" in doc

def notes_path(doc: papis.document.Document) -> str:
    """Get the path to the notes file corresponding to *doc*.

    If the document does not have attached notes, a filename is constructed (using
    the :ref:`config-settings-notes-name` setting) in the document's main folder.

    :returns: a absolute filename that corresponds to the attached notes for
        *doc* (this file does not necessarily exist).
    """
    if not has_notes(doc):
        notes_name = papis.format.format(papis.config.getstring("notes-name"), doc,
                                         default="notes.tex")
        doc["notes"] = papis.utils.clean_document_name(notes_name)
        papis.api.save_doc(doc)

    return os.path.join(doc.get_main_folder() or "", doc["notes"])

def notes_path_ensured(doc: papis.document.Document) -> str:
    """Get the path to the notes file corresponding to *doc* or create it if
    it does not exist.

    If the notes do not exist, a new file is created using :func:`notes_path`
    and filled with the contents of the template given by the
    :ref:`config-settings-notes-template` configuration option.

    :returns: an absolute filename that corresponds to the attached notes for *doc*.
    """
    notespath = notes_path(doc)

    if not os.path.exists(notespath):
        templatepath = os.path.expanduser(papis.config.getstring("notes-template"))

        template = ""
        if os.path.exists(templatepath):
            with open(templatepath, encoding="utf-8") as fd:
                try:
                    template = papis.format.format(fd.read(), doc)
                except papis.format.FormatFailedError as exc:
                    logger.error("Failed to format notes template at '%s'.",
                                 templatepath, exc_info=exc)

        with open(notespath, "w+", encoding="utf-8") as fd:
            fd.write(template)

    return notespath


import logging,json,findspark
findspark.init()
import pyspark
from time import sleep
from pathlib import Path
from pyspark.sql import SparkSession,Window
import pyspark.sql.functions as F
from pyspark.sql.types import *
import matplotlib.pyplot as plt
import seaborn as sns
#%matplotlib inline

logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
prefix_check = ["GB","US","CA","FR","JP","KR","RU","DE","BR"]
output_dir = f"../output/"
data_dir = f"../data/"

def create_spark_session(description):
    logger.info("Reading Spark object")
    spark = SparkSession.builder.appName(description).getOrCreate()
    logger.info("Spark read ok")

    return spark

def process_csv(spark,csv):

    prefix  = Path(csv).name[0:2]

    csv_schema = StructType([StructField("video_id",StringType(),False),
                            StructField("title",StringType(),True),
                            StructField("publishedAt",DateType(),True),
                            StructField("channelId",StringType(),True),
                            StructField("channelTitle",StringType(),True),
                            StructField("categoryId",ShortType(),True),
                            StructField("trending_date",DateType(),True),
                            StructField("tags",StringType(),True),
                            StructField("view_count",LongType(),True),
                            StructField("likes",LongType(),True),
                            StructField("dislikes",LongType(),True),
                            StructField("comment_count",LongType(),True),
                            StructField("comments_disabled",BooleanType(),True),
                            StructField("ratings_disabled",BooleanType(),True),
                            StructField("description",StringType(),True),
                            StructField("country",StringType(),True)]
                        )

    logger.info("Reading csv to dataframe using csv_schema")
    df_pyspark = spark.read.csv(csv, header=True,schema=csv_schema)

    logger.info("transforming dataframe")

    ############################################################
    #add trend count to df
    ############################################################
    trending_count = df_pyspark.groupby("video_id").count().withColumnRenamed("count","trendCount")

    df_pyspark = df_pyspark.join(trending_count,"video_id").select(df_pyspark["*"],trending_count["trendCount"])

    #use window function to drop all duplicate row
    win = Window.partitionBy("video_id").orderBy("trending_date")

    df_pyspark = df_pyspark.withColumn('rank', F.row_number().over(Window.partitionBy(df_pyspark.video_id).orderBy(df_pyspark.trending_date.desc())))\
                        .filter(F.col('rank') == 1)\
                        .drop("rank")

    ############################################################
    #read category name from json and add to df
    ############################################################
    logger.info("read category json")
    category_id = {}
    try:
        with open("../data/"+prefix+"_category_id.json","r") as f:
            category = json.load(f)
    except:
        logger.info("Error at loading/reading json, check if json file exist")
        return
    else:
        for cat in category["items"]:
            category_id[int(cat["id"])] = cat["snippet"]["title"]

    cat = list(map(list, category_id.items()))

    category = spark.createDataFrame(cat, ["categoryId", "categoryName"])

    logger.info("add category name to df")

    df_pyspark = df_pyspark.join(category,"categoryId").select(df_pyspark["*"], category["categoryName"])

    logger.info("create cat like")

    cat_like = df_pyspark.groupby("categoryName")\
                        .sum("likes","dislikes")\
                        .withColumnRenamed("sum(likes)","totalLikes")\
                        .withColumnRenamed("sum(dislikes)","totalDislikes")\
                        .withColumn("totalDislikes",F.col("totalDislikes")+1)\
                        .withColumn("liketoDislike",((F.col("totalLikes"))/(F.col("totalDislikes"))).cast(IntegerType()))\
                        .orderBy("totalLikes",ascending=False)

    logger.info("to pandas")

    spark.conf.set("spark.sql.execution.arrow.pyspark.enabled", "true")

    ############################################################
    #create graph for like to dislike vidoe category
    ############################################################
    logger.info("start plotting most_liked_category")

    plt.figure(figsize=(16, 14))
    plt.title("Like to dislike ratio for each category in " + prefix,fontsize = 24,fontweight = "bold")
    plt.ticklabel_format(style="plain",useOffset=False)
    sns.color_palette("light:b", as_cmap=True)
    ax = sns.barplot(x="liketoDislike", y="categoryName", data=cat_like.toPandas(), color="c")
    ax.set_xlabel ("Likes-Dislikes Ratio",fontsize = 16)
    ax.set_ylabel ("Category Name",fontsize = 16)

    plt.savefig(output_dir+prefix+"_most_liked_category.png")

    ############################################################
    #create graph for most trending category base on view count
    ############################################################
    logger.info("start plotting most_trending _category")

    cat_trend = df_pyspark.groupby("categoryName").count().orderBy("count",ascending=False)
    plt.clf()
    plt.figure(figsize=(16, 14))
    plt.title("Most trending video category in " + prefix,fontsize = 24,fontweight = "bold")
    plt.ticklabel_format(style='plain',useOffset=False)
    sns.color_palette("flare", as_cmap=True)
    ax = sns.barplot(x="count", y="categoryName", data=cat_trend.toPandas(),orient='h')
    ax.set_xlabel ("Number of trending video",fontsize = 16)
    ax.set_ylabel ("Category Name",fontsize = 16)

    plt.savefig(output_dir+prefix+"_most_trending_category.png")

if __name__ == "__main__":

    csv_list = [i for i in (Path(data_dir).glob("*.csv"))]

    session = create_spark_session("nlp word cloud generation")
    for csv in csv_list:
        if csv.name[0:2] not in prefix_check:
            logger.info("Inappropriate file name in folder, abort now")
            break
        else:
            process_csv(session,str(csv))
            sleep(5)


import os
import shutil
import subprocess
import tempfile
import gevent

from rq import (
    Connection,
    Queue,
)

from utils import (
    all_file_paths_exist,
    copy_or_symlink,
    try_run_command,
    DockerUtils,
    PacmanPackageCache,
    remove
)

from . import (
    RedisHash,
    get_build_object,
    get_pkg_object,
    status,
    get_repo_object
)

logger = status.logger
doc_util = DockerUtils(status)
doc = doc_util.doc

pkg_cache_obj = PacmanPackageCache()

with Connection(status.db):
    repo_queue = Queue('update_repo')

class TransactionMeta(RedisHash):
    """
    This is the base class for `Transaction`(s). It simply sets up the attributes
    which are stored in redis so they can be properly accessed. This class should
    not be used directly.

    Args:
        See `Transaction` docstring.

    Attributes:
        See `Transaction` docstring.
    """

    _main_repo = get_repo_object('antergos', 'x86_64')
    _staging_repo = get_repo_object('antergos-staging', 'x86_64')
    _main_repo32 = get_repo_object('antergos', 'i686')
    _staging_repo32 = get_repo_object('antergos-staging', 'i686')

    attrib_lists = dict(
        string=['building', 'start_str', 'end_str', 'initiated_by', 'gh_sha_before', 'gh_sha_after', 'gh_patch'],
        bool=['is_running', 'is_finished', 'sync_pkgbuilds_only'],
        int=['tnum'],
        list=['queue'],
        set=['packages', 'builds', 'completed', 'failed', 'generated_pkgs'],
        path=['base_path', 'path', 'result_dir', 'cache', 'cache_i686', 'upd_repo_result']
    )

    def __init__(self, packages=None, tnum=None, base_path='/var/tmp/antbs', namespace='antbs',
                 prefix='trans', repo_queue=None):
        if not packages and not tnum:
            raise ValueError('At least one of [packages, tnum] required.')
        elif packages and tnum:
            raise ValueError('Only one of [packages, tnum] can be given, not both.')

        the_tnum = tnum
        if not tnum:
            the_tnum = self.db.incr('antbs:misc:tnum:next')

        super().__init__(namespace=namespace, prefix=prefix, key=the_tnum)

        self.__namespaceinit__()

        self._repo_queue = repo_queue
        self._internal_deps = []
        self._build_dirpaths = {}
        self._pkgvers = {}

        if not self or not self.tnum:
            self.tnum = the_tnum
            self.base_path = base_path
            self.cache = pkg_cache_obj.cache
            self.cache_i686 = pkg_cache_obj.cache_i686

            if packages:
                packages = [p for p in packages if p]

                for pkg in packages:
                    self.packages.add(pkg)

        for pkg in self.packages:
            self._build_dirpaths[pkg] = {'build_dir': '', '32bit': '', '32build': ''}
            self._pkgvers[pkg] = ''

class Transaction(TransactionMeta):
    """
    This class represents a single "build transaction" throughout the app. It is used
    to get/set transaction data from/to the database. A transaction is comprised
    of one or more builds. When a new transaction is initialized it creates its own build
    directory which it will delete once all builds are completed. This allows for
    build concurrency through multiple transactions and can be easily scaled as needed.

        Args:
            packages (list): Names of packages to build. This creates a new `Transaction`.
            tnum (int): Get an existing `Transaction` identified by its `tnum`.

        Attributes:
            tnum (int): This transaction's number or id if you prefer calling it that.
            base_path (str): Absolute path to the top-level build directory (for all transactions).
            path (str): Absolute path to this transaction's build directory.
            builds (list): This transaction's builds (list of bnums)
            is_running (bool): Whether or not the transaction is currently running.
            is_finished (bool): Whether or not the transaction is done (regardless of results)
            building (str): The name of the package currently building.
            start_str (str): The datetime string for when this transaction started.
            end_str (str): The datetime string for when this transaction ended.
            completed (list): Builds that completed successfully (list of bnums).
            failed (list): Builds that failed (list of bnums).
            internal_deps (list): List of packages that depend on package(s) in this transaction.

        Raises:
            ValueError: If both `packages` and `tnum` are Falsey.
    """

    def start(self):
        if self._repo_queue is None:
            logger.debug('self._repo_queue is: %s', self._repo_queue)
            raise AttributeError('_repo_queue is required to start a transaction.')

        status.current_status = 'Initializing build transaction.'
        self.is_running = True

        status.transactions_running.append(self.tnum)
        self.setup_transaction_directory()

        status.current_status = 'Processing packages.'

        self.process_packages()

        if self.sync_pkgbuilds_only:
            return

        status.current_status = 'Cleaning pacman package cache.'

        if len(status.transactions_running) == 1:
            PacmanPackageCache().maybe_do_cache_cleanup()

        while self.queue:
            pkg = self.queue.lpop()
            build_dir = self.get_build_directory(pkg)

            if not build_dir:
                raise RuntimeError('build_dir cannot be None.')

            pkg_obj = get_pkg_object(name=pkg)
            bld_obj = get_build_object(pkg_obj=pkg_obj, tnum=self.tnum, trans_obj=self)

            if pkg_obj.is_iso:
                self.fetch_and_compile_translations(
                    translations_for=["cnchi_updater", "antergos-gfxboot"]
                )
                result = bld_obj.start(pkg_obj)
            else:
                bld_obj = self.setup_build_directory(bld_obj, build_dir)
                result = bld_obj.start(pkg_obj)

            if result in [True, False]:
                blds = pkg_obj.builds
                total = len(blds)

                if total > 0:
                    success = len([x for x in blds if x in status.completed])
                    failure = len([x for x in blds if x in status.failed])

                    if success > 0:
                        success = 100 * success / total

                    if failure > 0:
                        failure = 100 * failure / total

                    pkg_obj.success_rate = success
                    pkg_obj.failure_rate = failure

                if result is True:
                    if not pkg_obj.is_iso:
                        gevent.sleep(2)
                        self.move_files_to_staging_repo(bld_obj)
                        repo_queue.enqueue_call(self._staging_repo.update_repo)
                        repo_queue.enqueue_call(self._staging_repo32.update_repo)
                        gevent.sleep(12)

                    self.completed.append(bld_obj.bnum)
                    doc_util.do_docker_clean(pkg_obj.name)

                elif result is False:
                    self.failed.append(bld_obj.bnum)

            status.now_building.remove(bld_obj.bnum)

        self.is_running = False
        self.is_finished = True
        status.transactions_running.remove(self.tnum)

        remove(self.path)

    def setup_transaction_directory(self):
        path = tempfile.mkdtemp(prefix='{0}_'.format(str(self.tnum)), dir=self.base_path)
        os.chmod(path, 0o777)
        self.result_dir = os.path.join(path, 'result')
        self.upd_repo_result = os.path.join(path, 'upd_result')
        self.path = os.path.join(path, 'antergos-packages', 'antergos')

        os.mkdir(self.result_dir, mode=0o777)
        os.mkdir(self.upd_repo_result, mode=0o777)

        try:
            subprocess.check_output(['/usr/bin/git', 'clone', status.gh_repo_url], cwd=path)
        except subprocess.CalledProcessError as err:
            raise RuntimeError(err.output)

    def get_build_directory(self, pkg):
        pbpath = None
        paths = [
            os.path.join(self.path, 'mate', pkg),
            os.path.join(self.path, 'cinnamon', pkg),
            os.path.join(self.path, 'liri', pkg),
            os.path.join(self.path, pkg)
        ]

        for p in paths:
            if os.path.exists(p):
                pbpath = p
                break
        else:
            raise RuntimeError('Unable to determine pb_path for {0}'.format(pkg))

        return pbpath

    def setup_build_directory(self, bld_obj, build_dir):

        self._build_dirpaths[bld_obj.pkgname].update({
            'build_dir': build_dir,
            '32bit': os.path.join(build_dir, '32bit'),
            '32build': os.path.join(build_dir, '32build'),
            'result': os.path.join(self.result_dir, bld_obj.pkgname)
        })

        for bdir, path in self._build_dirpaths[bld_obj.pkgname].items():
            if not os.path.exists(path):
                os.mkdir(path, mode=0o777)

        bld_obj.build_dir = self._build_dirpaths[bld_obj.pkgname]['build_dir']
        bld_obj._32bit = self._build_dirpaths[bld_obj.pkgname]['32bit']
        bld_obj._32build = self._build_dirpaths[bld_obj.pkgname]['32build']
        bld_obj.result_dir = self._build_dirpaths[bld_obj.pkgname]['result']

        return bld_obj

    def handle_special_cases(self, pkg, pkg_obj):
        if pkg not in ['cnchi-dev', 'cnchi']:
            return

        pkg_obj.prepare_package_source(self.get_build_directory(pkg))

        if 'cnchi' in pkg:
            logger.info('cnchi package detected.')
            status.current_status = 'Fetching latest translations for %s from Transifex.' % pkg
            logger.info(status.current_status)
            self.fetch_and_compile_translations(translations_for=["cnchi"], pkg_obj=pkg_obj)

    @staticmethod
    def move_files_to_staging_repo(bld_obj):
        file_count = len(bld_obj.generated_files)
        files_exist = bld_obj.generated_files and all_file_paths_exist(bld_obj.generated_files)

        if not files_exist or not (file_count % 2 == 0):
            logger.error(
                'Unable to move files to staging repo! files_exist is: %s file_count is: %s',
                files_exist,
                file_count
            )

        for pkg_file in bld_obj.generated_files:
            if 'i686' in pkg_file:
                continue

            fname = os.path.basename(pkg_file)
            staging_file = os.path.join(status.STAGING_64, fname)

            copy_or_symlink(pkg_file, status.STAGING_64, logger)
            bld_obj.staging_files.append(staging_file)

            if '-any.pkg' in pkg_file:
                src = os.path.basename(pkg_file)
                dst = '../i686/{}'.format(fname)

                success, res = try_run_command(
                    ['/bin/ln', '-srf', src, dst],
                    cwd=status.STAGING_64,
                    logger=logger
                )
                if not success:
                    logger.error(res)

        for pkg_file in bld_obj.generated_files:
            if 'x86_64' in pkg_file or '-any.pkg' in pkg_file:
                continue

            fname = os.path.basename(pkg_file)
            staging_file = os.path.join(status.STAGING_32, fname)

            copy_or_symlink(pkg_file, status.STAGING_32, logger)
            bld_obj.staging_files.append(staging_file)

    def process_packages(self):
        _pkgs = [p for p in self.packages if p]

        for pkg in self.packages:
            if not pkg:
                continue

            pbpath = self.get_build_directory(pkg)

            if not pbpath:
                raise RuntimeError('pbpath cannot be None.')

            pkg_obj = get_pkg_object(name=pkg)
            version = pkg_obj.get_version_str()

            if not version and not pkg_obj.is_monitored:
                self.packages.remove(pkg)
                logger.debug('Skipping cnchi-dev build: {0}'.format(pkg))
                continue

            pkg_obj.version_str = version
            self._pkgvers[pkg] = version

            if pkg_obj.is_split_package and pkg_obj.pkgname == pkg_obj.gh_path.rsplit('/')[1]:
                for split_pkg in pkg_obj.split_packages:
                    split_pkg_obj = get_pkg_object(split_pkg)
                    split_pkg_obj.version_str = version
                    split_pkg_obj.pkgver = pkg_obj.pkgver

            log_msg = 'Updating pkgver in database for {0} to {1}'.format(pkg, version)
            logger.info(log_msg)
            status.current_status = log_msg

            depends = pkg_obj.get_deps()

            intersect = list(set(depends) & set(_pkgs))
            logger.debug((depends, intersect))
            if depends and len(intersect) > 0:
                self._internal_deps.append((pkg, intersect))
            else:
                self._internal_deps.append((pkg, []))

            self.handle_special_cases(pkg, pkg_obj)

        status.current_status = 'Using package dependencies to determine build order.'
        if self._internal_deps:
            for name in self.determine_build_order(self._internal_deps):
                if name not in self.queue:
                    self.queue.append(name)

        for pkg in self.packages:
            if pkg not in self.queue:
                self.queue.append(pkg)

    def fetch_and_compile_translations(self, translations_for=None, pkg_obj=None):
        """
        Get and compile translations from Transifex.

        :param (list) translations_for:
        :param (Package) pkg_obj:

        """

        if pkg_obj is None:
            name = ''
        else:
            name = pkg_obj.name or pkg_obj.pkgname

        pbdir = self.get_build_directory(name)
        dest_dir = os.path.join(pbdir, 'po')

        trans = {
            "cnchi": {
                'trans_dir': status.CNCHI_TRANSLATIONS_DIR,
                'trans_files_dir': os.path.join(status.CNCHI_TRANSLATIONS_DIR, '/translations/antergos.cnchi'),
                'dest_dir': dest_dir
            },
            "cnchi_updater": {
                'trans_dir': status.ISO_TRANSLATIONS_DIR,
                'trans_files_dir': os.path.join(status.ISO_TRANSLATIONS_DIR, '/translations/antergos.cnchi_updaterpot'),
                'dest_dir': os.path.join(status.ISO_TRANSLATIONS_DESTDIR, '/cnchi_updater')
            },
            "antergos-gfxboot": {
                'trans_dir': status.ISO_TRANSLATIONS_DIR,
                'trans_files_dir': os.path.join(status.ISO_TRANSLATIONS_DIR, '/translations/antergos.antergos-gfxboot'),
                'dest_dir': os.path.join(status.ISO_TRANSLATIONS_DESTDIR, '/antergos-gfxboot')
            }
        }

        for trans_for in translations_for:

            if not os.path.exists(trans[trans_for]['dest_dir']):
                os.mkdir(trans[trans_for]['dest_dir'])
            try:

                output = subprocess.check_output(['tx', 'pull', '-a', '--minimum-perc=50'],
                                                 cwd=trans[trans_for]['trans_dir'])

                for r, d, f in os.walk(trans[trans_for]['trans_files_dir']):
                    for tfile in f:
                        if trans_for in ['cnchi', 'antergos-gfxboot']:
                            tfile = os.path.join(r, tfile)
                            logger.debug(
                                'Copying %s to %s' % (tfile, trans[trans_for]['dest_dir']))
                            shutil.copy(tfile, trans[trans_for]['dest_dir'])
                        elif 'cnchi_updater' == trans_for:
                            mofile = tfile[:-2] + 'mo'
                            subprocess.check_call(['msgfmt', '-v', tfile, '-o', mofile],
                                                  cwd=trans[trans_for]['trans_files_dir'])
                            os.rename(os.path.join(trans[trans_for]['trans_files_dir'], mofile),
                                      os.path.join(trans[trans_for]['dest_dir'], mofile))

            except subprocess.CalledProcessError as err:
                logger.error(err.output)
            except Exception as err:
                logger.error(err)

    @staticmethod
    def determine_build_order(source):
        """
        Performs a topological sort on elements. This determines the order in which
        packages must be built based on internal (to this transaction) dependencies.

        Args:
            source (list): A list of ``(name, [list of dependancies])`` pairs.

        Returns:
            A list of names, with dependancies listed first.

        Raises:
            ValueError: When cyclic or missing dependancy detected.

        """
        # copy deps so we can modify set in-place
        pending = [(name, set(deps)) for name, deps in source]
        emitted = []
        try:
            while pending:
                next_pending = []
                next_emitted = []

                for entry in pending:
                    name, deps = entry
                    # remove deps we emitted last pass
                    deps.difference_update(emitted)

                    if deps:
                        # still has deps? recheck during next pass
                        next_pending.append(entry)
                    else:
                        # no more deps? time to emit
                        yield name
                        emitted.append(name)
                        # remember what we emitted for difference_update() in next pass
                        next_emitted.append(name)

                if not next_emitted:
                    # all entries have unmet deps, one of two things is wrong...
                    logger.error('cyclic or missing dependancy detected: %r', next_pending)
                    names = [n for n, d in source]
                    deps = [d for n, d in source]
                    missing = [m for d in deps for m in d if m not in names]
                    logger.error(names)
                    logger.error(deps)
                    logger.error(missing)

                    raise ValueError
                pending = next_pending
                emitted = next_emitted
        except ValueError as err:
            logger.error(err)

def get_trans_object(packages=None, tnum=None, repo_queue=None):
    """
    Gets an existing transaction or creates a new one.

    Args:
        packages (list): Create a new transaction with these packages.
        tnum (int): Get an existing transaction identified by `tnum`.

    Returns:
        Transaction: A fully initiallized `Transaction` object.

    Raises:
        ValueError: If both `packages` and `tnum` are Falsey or Truthy.

    """
    if not any([packages, tnum]):
        raise ValueError('At least one of [packages, tnum] required.')
    elif all([packages, tnum]):
        raise ValueError('Only one of [packages, tnum] can be given, not both.')

    trans_obj = Transaction(packages=packages, tnum=tnum, repo_queue=repo_queue)

    return trans_obj


#https://github.com/python-visualization/folium

import folium
import pandas

df = pandas.read_csv("Volcanoes-USA.txt")

map = folium.Map(location=[df['LAT'].mean(), df['LON'].mean()], zoom_start=12, tiles="Stamen Terrain") #latitude and longitude

def color(elev):
    min_color = int(min(df['ELEV']))
    step = int((max(df['ELEV']) - min_color) / 3)
    if elev in range(min_color, min_color + step):
        return 'green'
    elif elev in range(min_color + step, min_color + 2 * step):
        return 'orange'
    else:
        return 'red'

for lat, lon, name, elev in zip(df['LAT'], df['LON'], df['NAME'], df['ELEV']):
    folium.Marker(location=[lat, lon], popup=name, icon=folium.Icon(color=color(elev))).add_to(map)

map.save("test_map_2.html")


import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import os

def save_images_result(exp_path, csv_path):
    if not os.path.exists(exp_path):
        os.makedirs(exp_path)

    accuracies, class_recall, poison_rate = read_csv(csv_path=csv_path, attacker_label=None, poison_label=4)
    draw_results(accuracies, class_recall, poison_rate, exp_path)

def read_csv(csv_path, attacker_label, poison_label):

    accuracies = list()
    class_recall = list()
    poison_rate = list()

    df = pd.read_csv(csv_path, header=None)
    for idx, row in df.iterrows():
        accuracy = 0.0
        for i in range(10):
            if i != poison_label:
                accuracy += row[i + 12]
        accuracies.append(accuracy/9)
        class_recall.append(row[poison_label + 12])
        poison_rate.append(row[22]/1000)

    return accuracies, class_recall, poison_rate

def draw_results(accuracies, class_recall, poison_rate, exp_path):
    round = [i for i in range(200)]

    # Main task and Poison task Accuracy
    plt.figure(figsize=(10, 5))
    plt.plot(round, accuracies, label="Main task")
    plt.plot(round, class_recall, label="Poison task")
    plt.xlabel('Round')
    plt.ylabel('Accuracy')
    plt.legend(loc='best')
    plt.savefig(exp_path + '/class_recall.png')

    # Main task accuracy and Poison success rate
    plt.figure(figsize=(10, 5))
    plt.plot(round, accuracies, label="Main task")
    plt.plot(round, poison_rate, label="Poison task")
    plt.xlabel('Round')
    plt.ylabel('Accuracy')
    plt.legend(loc='best')
    plt.savefig(exp_path + '/poison_success_rate.png')

if __name__ == "__main__":
    save_images_result('experiment/multi_round_attack/exp_1', '3000_results.csv')


import sys
import csv
import pandas as pd
import os
from itertools import combinations
import functools
from pathlib import Path

VIEWS_THRESHOLD = 0
OVERLAP_THRESHOLD = 5

@functools.lru_cache(None)
def LoadStreamerViewers(streamer):
    v = pd.read_csv(f"../DataCollection/data/{streamer}.csv", names=['username','n'])
    v = v[v['n'] > VIEWS_THRESHOLD]['username']
    if(len(v) < OVERLAP_THRESHOLD): return set()
    return set(v)

def CreateOverlapDict(streamers):
    overlaps = {}
    combo = combinations(streamers,2)

    print(f"Combining {len(streamers)} streamers...")
    for s1,s2 in combo:
        v1 = LoadStreamerViewers(s1)
        v2 = LoadStreamerViewers(s2)

        overlap = len(v1 & v2)
        if overlap < OVERLAP_THRESHOLD: continue
        print(f"{s1} and {s2} have {overlap} viewers in common")
        if s1 not in overlaps: overlaps[s1] = {}
        if s2 not in overlaps[s1]: overlaps[s1][s2] = {}
        overlaps[s1][s2] = overlap
    return overlaps

def GenerateGephiData(dict):
    dir_path = Path().absolute()
    data_path = f'{dir_path}/gephi/Data.csv'
    with open(data_path, 'w') as csvfile:
        writer = csv.writer(csvfile)
        writer.writerow(["Source", "Target", "Weight"])
        for key, value in dict.items():
            nodeA = key
            for node, count in value.items():
                nodeB = node
                writer.writerow([nodeA, nodeB, count])

def GenerateGephiLabels(streamers):
    dir_path = Path().absolute()
    labels_path = f'{dir_path}/gephi/Labels.csv'
    print("Generating Labels...")
    with open(labels_path, 'w') as csvfile:
        writer = csv.writer(csvfile)
        writer.writerow(["ID", "Label", "Count"])
        for s in streamers:
            v = set(open(f'{dir_path}/../DataCollection/data/{s}.csv').read().splitlines())
            writer.writerow([s, s, len(v)])

print("Reading streamers")
dir_path = Path().absolute()
streamers = [ f.replace('.csv','') for f in os.listdir(f'{dir_path}/../DataCollection/data') ]
print("Creating overlap dict")
d = CreateOverlapDict(streamers)
GenerateGephiData(d)
GenerateGephiLabels(streamers)


# -*- coding: utf-8 -*-
"""
=========================================================
Compare CO spectrum from the GEISA and HITRAN database
=========================================================

GEISA Database has been newly implemented in RADIS 0.13 release on May 15, 2022. This is among
the very first attempts to compare the spectra generated from the two databases.

Auto-download and calculate CO spectrum from the GEISA database, and the HITRAN database.

Output should be similar, but not exactly! By default these two databases provide different
broadening coefficients. However, the Einstein coefficients & linestrengths should be
approximately the same, therefore the integrals under the lines should be similar.

You can see it by running the code below.

For your interest, GEISA and HITRAN lines can be downloaded and accessed separately using
:py:func:`~radis.io.geisa.fetch_geisa` and :py:func:`~radis.io.hitran.fetch_hitran`

"""
import astropy.units as u

from radis import calc_spectrum, plot_diff

conditions = {
    "wmin": 2002 / u.cm,
    "wmax": 2300 / u.cm,
    "molecule": "CO",
    "pressure": 1.01325,  # bar
    "Tgas": 1000,  # K
    "mole_fraction": 0.1,
    "path_length": 1,  # cm
    "verbose": True,
}

s_geisa = calc_spectrum(**conditions, databank="geisa", name="GEISA's CO")

s_hitran = calc_spectrum(
    **conditions,
    databank="hitran",
    name="HITRAN's CO",
)

"""

In :py:func:`~radis.io.geisa.fetch_geisa`, you can choose to additionally plot the
absolute difference (method='diff') by default, or the ratio (method='ratio'), or both.

"""

plot_diff(s_geisa, s_hitran, method=["diff", "ratio"])