import io
import zipfile

import pandas as pd
import pytest
import requests
from bs4 import BeautifulSoup

from oda_data import config
from oda_data.get_data import common

def test__checktype():
    expected_int = [1, 2, 3]
    expected_float = [1.0, 2.0, 3.0]

    tests = [[1, 2, 3], ["1", "2", "3"], [1.0, 2.0, 3.0]]

    for test in tests:
        assert common._checktype(test, int) == expected_int
        assert common._checktype(test, float) == expected_float

    assert common._checktype("1", int) == [1]
    assert common._checktype(1, int) == [1]
    assert common._checktype(1.0, int) == [1]
    with pytest.raises(ValueError):
        common._checktype((1, 2, 3), int)

def test_check_integers():
    tests = [[1, 2, 3], ["1", "2", "3"], [1.0, 2.0, 3.0]]
    for test in tests:
        assert common.check_integers(test) == [1, 2, 3]

    assert common.check_integers(range(2020, 2022)) == [2020, 2021]

def test_get_zip():
    """This function is essentially just a wrapper around requests.get"""
    # TODO: work on a test
    assert True

def test__serialise_content():
    with open(config.OdaPATHS.test_files / "zip_response", "rb") as f:
        response = f.read()

    mock_response = requests.Response()
    mock_response._content = response

    assert isinstance(common._serialise_content(mock_response), io.BytesIO)

def test__extract_zip():
    with open(config.OdaPATHS.test_files / "zip_response", "rb") as f:
        response = f.read()

    mock_response = requests.Response()
    mock_response._content = response

    serialised = common._serialise_content(mock_response)

    file = common._extract_zip(serialised, "Table1_Data.csv")

    assert isinstance(file, zipfile.ZipExtFile)

def test__raw2df():
    file_comma = config.OdaPATHS.test_files / "comma_df_raw.csv"
    file_tab = config.OdaPATHS.test_files / "tab_df_raw.csv"

    df_comma = common._raw2df(file_comma, sep=",", encoding="ISO-8859-1")

    with pytest.raises(pd.errors.ParserError):
        common._raw2df(file_tab, sep="|", encoding="ISO-8859-1")

    assert isinstance(df_comma, pd.DataFrame)
    assert len(df_comma.columns) > 3

def test_read_zip_content():
    with open(config.OdaPATHS.test_files / "zip_response", "rb") as f:
        response = f.read()

    mock_response = requests.Response()
    mock_response._content = response

    result = common.read_zip_content(
        request_content=mock_response, file_name="Table1_Data.csv", priority_sep=","
    )

    assert isinstance(result, pd.DataFrame)

def test__links_dict():
    with open(config.OdaPATHS.test_files / "response_multi", "r") as f:
        response = f.read()

    soup = BeautifulSoup(response, "html.parser")
    results_set = soup.find_all("a")

    links = common._links_dict(results_set)

    assert isinstance(links, dict)
    assert len(links) == 21
    assert links.get(2021) is not None


#-*- coding: utf-8 -*-

import numpy as np
import pydotplus.graphviz as gv

class SimpleWeb(object):
    def __init__(self, nombreSommet):
        """ on crée des noeuds lors de l'initialisation"""
        self.nombreSommet = nombreSommet
        self.listeSommet = []
        self.arcExistant = set()
        for i in range(0,nombreSommet):
            n = Node(i)
            self.listeSommet.append(n)
        self.matriceProba = np.zeros((nombreSommet, nombreSommet))
        self.nb_pas = None
        self.fichier = None

    def addArc(self,id_n1,id_n2):
        """ id_n1(tail) : int, id_n2(head) : int -> ajoute les arc aux sommets référencés"""
        if id_n1 == id_n2:
            raise Exception("meme noeud")
        if (id_n1, id_n2) in self.arcExistant:
            raise Exception("l'arc existe deja")
        self.arcExistant.add((id_n1,id_n2))
        n1 = self.listeSommet[id_n1]
        n2 = self.listeSommet[id_n2]
        #print("on ajoute un arc de {} vers {}".format(n1,n2))
        a = Arc(id_n1,id_n2)
        n2.ajoute_arc_entrant(a)
        n1.ajoute_arc_sortant(a)
        self.updateProbas()

    def updateProbas(self):
        """ on parcourt tous les noeuds et incremente a partir des arcs entrants"""
        for i in range(0,self.nombreSommet):
            n = self.listeSommet[i]
            #le noeud n'a aucun arc
            if len(n.arcSortant) == 0:
                self.matriceProba[i][i] = 1
            #si il a un nouvelle arc, on supprime l'arc pointant vers lui meme
            if len(n.arcSortant) == 1 and self.matriceProba[i][i] == 1:
                self.matriceProba[i][i] = 0
            for arc in n.arcSortant:
                head = arc.head
                tail = arc.tail
                arc.proba = 1 / (len(n.arcSortant)*1.0)
                self.matriceProba[tail][head] = arc.proba

    def __str__(self):
        s =  "MatriceProba : {} \n".format(self.matriceProba)
        for i in range(self.nombreSommet):
            n = self.listeSommet[i]
            if not(n.arcSortant_vide()):
                s += n.__str__()
        return s

    def writeGraph(self, fichier):
        ''' fichier : nom du fichier dans le repertoire courant '''
        graph = gv.Dot(graph_type='digraph')
        for node in self.listeSommet:
            graph.add_node(gv.Node(node.id_node))
        for node in self.listeSommet:
            for arc in node.arcSortant:
                graph.add_edge(gv.Edge(node.id_node, arc.head, label=arc.proba))
        graph.write_png(fichier)

    def nextStep(self,pi_t):
        """ np.array -> np.array"""
        return np.dot(pi_t,self.matriceProba)

    def convergence_p(self, iterMax, seuil):
        """ p : matriceProba"""
        matricePuissante = np.copy(self.matriceProba)
        cpt = 0
        epsilon = 1
        while epsilon > seuil and cpt < iterMax:
            newPuissance = np.dot(matricePuissante, self.matriceProba)
            dif = abs(matricePuissante - newPuissance)
            epsilon = np.amax(dif)
            if self.nb_pas != None and self.fichier != None:
                if cpt % self.nb_pas == 0 and cpt != 0:
                    self.ecrit(cpt, epsilon)
                    #print("epsilon : {}".format(epsilon))
            matricePuissante = newPuissance
            cpt += 1

        print("\n------- PUISSANCES -------")
        print("iterations : {}".format(cpt))
        print("epsilon : {}\n".format(epsilon))
        return matricePuissante

    def trace(self, nb_pas, fichier):
        self.nb_pas = nb_pas
        self.fichier = fichier
        #on supprime l'ancienne valeur
        f = open(self.fichier, 'w')
        f.close()

    def ecrit(self, nb, epsilon):
        #on crée les nouvelles valeurs
        f = open(self.fichier, 'a')
        e = (str)(epsilon)
        f.write(str(nb) + ' ' + e + '\n')
        f.close()

class Node(object):
    def __init__(self,id_node):
        """ int * Arc[] * Arc[] -> void
        si un noeud n'a pas d'arc sortant, on place un arc vers lui meme avec une proba 1 """
        self.arcSortant = []
        self.arcEntrant = []
        self.id_node = id_node

    def arcSortant_vide(self):
        if self.arcSortant == []:
            return True
        return False

    def ajoute_arc_sortant(self,arc):
        self.arcSortant.append(arc)

    def ajoute_arc_entrant(self,arc):
        self.arcEntrant.append(arc)

    def __str__(self):
        s =  "Noeud numero : {} \n ".format(self.id_node)
        #print("liste des arc sortant : ",self.arcSortant)
        if self.arcSortant != []:
            s += "Arc sortant : \n"
            for arc in self.arcSortant:
                s += '\t' + arc.__str__() + '\n'
        return s

class Arc(object):
    def __init__(self,tail, head, proba=0):
        self.tail = tail
        self.head = head
        self.proba = proba

    def __str__(self):
        return "Arc du noeud : {} vers le noeud {} avec une probabilité : {}".format(self.tail, self.head,self.proba)


import torch
import torch.nn as nn
import torch.optim as optim
import pandas as pd
import zarr
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error

# 1. Data Preparation
def read_cf_pair(dt, n):
    start, end = dt.index_range[n]
    timestamps = dt.timestamp[start:end]
    t = (timestamps-timestamps[0]) # read time and convert it to second

    lead_centroid = dt.lead_centroid[start: end]
    follow_centroid = dt.follow_centroid[start: end]
    follow_velocity = dt.follow_velocity[start: end]
    lead_velocity = dt.lead_velocity[start: end]
    follow_acceleration = dt.follow_acceleration[start: end]
    lead_acceleration = dt.lead_acceleration[start: end]
    # size_lead = dt.lead_size[n] # this is for HV
    size_lead = 4.8
    size_follow = dt.follow_size[n]

    return lead_centroid, follow_centroid, follow_velocity, follow_acceleration, lead_acceleration, lead_velocity, t, size_lead, size_follow

# For easier data manipulation, we convert the data into a pandas dataframe
# x_lead, x_follow, v_follow, a_follow, a_lead, v_lead, t, size_lead, size_follow = read_cf_pair(data, 120)

zarr_data = zarr.open('../data/valAV.zarr', mode='r')
data = pd.DataFrame(data=dict(zip(['x_lead', 'x_follow', 'v_follow', 'a_follow', 'a_lead', 'v_lead', 't', 'size_lead', 'size_follow'], read_cf_pair(zarr_data, 120))))

# Calculate spacing and relative speed
data['spacing'] = data['x_lead'] - data['x_follow'] - data['size_lead']
data['relative_speed'] = data['v_lead'] - data['v_follow']

# Select features and target
# features = data[['spacing', 'relative_speed', 'v_follow', 'v_lead', 'a_follow' ,'a_lead', 'x_lead']]
features = data[['spacing', 'relative_speed']]
target = data['a_follow']

# Normalize data
scaler = MinMaxScaler()
features = scaler.fit_transform(features)

# Split into training and testing sets
X_train, X_test, y_train, y_test = train_test_split(features, target, test_size=0.2, random_state=42)

# Convert data to PyTorch tensors
X_train_tensor = torch.FloatTensor(X_train).unsqueeze(2)
y_train_tensor = torch.FloatTensor(y_train.values)
X_test_tensor = torch.FloatTensor(X_test).unsqueeze(2)
y_test_tensor = torch.FloatTensor(y_test.values)

# 2. Model Definition
class CarFollowingLSTM(nn.Module):
    def __init__(self, input_dim, hidden_dim, num_layers, output_dim):
        super(CarFollowingLSTM, self).__init__()
        self.hidden_dim = hidden_dim
        self.num_layers = num_layers
        self.lstm = nn.LSTM(input_dim, hidden_dim, num_layers, batch_first=True)
        self.linear = nn.Linear(hidden_dim, output_dim)

    def forward(self, x):
        h0 = torch.zeros(self.num_layers, x.size(0), self.hidden_dim).requires_grad_()
        c0 = torch.zeros(self.num_layers, x.size(0), self.hidden_dim).requires_grad_()
        out, (hn, cn) = self.lstm(x, (h0.detach(), c0.detach()))
        out = self.linear(out[:, -1, :])
        return out

input_dim = 1
hidden_dim = 32
num_layers = 2
output_dim = 1

model = CarFollowingLSTM(input_dim=input_dim, hidden_dim=hidden_dim, num_layers=num_layers, output_dim=output_dim)

# 3. Training
loss_function = nn.MSELoss()
optimizer = torch.optim.Adam(model.parameters(), lr=0.01)

num_epochs = 100
for epoch in range(num_epochs):
    model.train()
    optimizer.zero_grad()
    outputs = model(X_train_tensor)
    loss = loss_function(outputs, y_train_tensor)
    loss.backward()
    optimizer.step()
    if epoch % 10 == 0:
        print(f'Epoch {epoch}, Loss: {loss.item()}')

# 4. Saving
torch.save(model.state_dict(), 'car_following_model.pth')

# 5. Evaluation
model.eval()
test_outputs = model(X_test_tensor)
test_loss = loss_function(test_outputs, y_test_tensor)
print(f'Test Loss: {test_loss.item()}')

y_pred = test_outputs.detach().numpy()
y_true = y_test_tensor.numpy()
mse = mean_squared_error(y_true, y_pred)
print(f'MSE on test set: {mse}')


class Location(object):
    def __init__(self, x, y):
        self.x = x
        self.y = y

    def move(self, deltaX, deltaY):
        return Location(self.x + deltaX, self.y + deltaY)

    def getX(self):
        return self.x

    def getY(self):
        return self.y

    def distFrom(self, other):
        ox = other.x
        oy = other.y
        xDist = self.x - ox
        yDist = self.y - oy
        return (xDist**2 + yDist**2)**0.5

    def __str__(self):
        return '<' + str(self.x) + ', ' + str(self.y) + '>'

class Field(object):
    def __init__(self):
        self.drunks = {}

    def addDrunk(self, drunk, loc):
        if drunk in self.drunks:
            raise ValueError('Duplicate drunk')
        else:
            self.drunks[drunk] = loc

    def getLoc(self, drunk):
        if drunk not in self.drunks:
            raise ValueError('Drunk not in field')
        return self.drunks[drunk]

    def moveDrunk(self, drunk):
        if drunk not in self.drunks:
            raise ValueError('Drunk not in field')
        xDist, yDist = drunk.takeStep()
        currentLocation = self.drunks[drunk]
        self.drunks[drunk] = currentLocation.move(xDist, yDist)

class Drunk(object):
    def __init__(self, name = None):
        self.name = name

    def __str__(self):
        return 'This drunk is named ' + self.name

import random

class UsualDrunk(Drunk):
    def takeStep(self):
        stepChoices = \
            [(0.0, 1.0), (0.0, -1.0), (1.0, 0.0), (-1.0, 0.0)]
        return random.choice(stepChoices)

class ColdDrunk(Drunk):
    def takeStep(self):
        stepChoices = \
            [(0.0, 0.9), (0.0, -1.1), (1.0, 0.0), (-1.0, 0.0)]
        return random.choice(stepChoices)

def walk(f, d, numSteps):
    start = f.getLoc(d)
    for s in range(numSteps):
        f.moveDrunk(d)
    return start.distFrom(f.getLoc(d))

def simWalks(numSteps, numTrials, dClass):
    Homer = dClass()
    origin = Location(0, 0)
    distances = []
    for t in range(numTrials):
        f = Field()
        f.addDrunk(Homer, origin)
        distances.append(round(walk(f, Homer, numSteps), 1))
    return distances

def drunkTest(walkLengths, numTrials, dClass):
    for numSteps in walkLengths:
        distances = simWalks(numSteps, numTrials, dClass)
        print(dClass.__name__, 'random walk of', numSteps, 'steps')
        print(' Mean = ', round(sum(distances)/len(distances), 4))
        print(' Max = ', max(distances), 'Min = ', min(distances))

def simAll1(drunkKinds, walkLengths, numTrials):
    for dClass in drunkKinds:
        drunkTest(walkLengths, numTrials, dClass)

class styleIterator(object):
    def __init__(self, styles):
        self.index = 0
        self.styles = styles

    def nextStyle(self):
        result = self.styles[self.index]
        if self.index == len(self.styles) - 1:
            self.index = 0
        else:
            self.index += 1
        return result

def simDrunk(numTrials, dClass, walkLengths):
    meanDistances = []
    for numSteps in walkLengths:
        print('Starting simulation of', numSteps, 'steps')
        trials = simWalks(numSteps, numTrials, dClass)
        mean = sum(trials)/len(trials)
        meanDistances.append(mean)
    return meanDistances

import matplotlib
matplotlib.use('TkAgg')
import matplotlib.pyplot as plt

def simAll(drunkKinds, walkLengths, numTrials):
    styleChoice = styleIterator(('m-', 'b--', 'g-.'))
    for dClass in drunkKinds:
        curStyle = styleChoice.nextStyle()
        print('Starting simulation of', dClass.__name__)
        means = simDrunk(numTrials, dClass, walkLengths)
        plt.plot(walkLengths, means, curStyle, label=dClass.__name__)

    plt.title('Mean Distance from Origin (' + str(numTrials) + ' trials)')
    plt.xlabel('Number of steps')
    plt.ylabel('Distance from Origin')
    plt.legend(loc = 'best')
    plt.show()

def getFinalLocs(numSteps, numTrials, dClass):
    locs = []
    d = dClass()
    for t in range(numTrials):
        f = Field()
        f.addDrunk(d, Location(0,0))
        for s in range(numSteps):
            f.moveDrunk(d)
        locs.append(f.getLoc(d))
    return locs

import numpy as np
def plotLocs(drunkKinds, numSteps, numTrials):
    styleChoice = styleIterator(('k+', 'r^', 'mo'))
    for dClass in drunkKinds:
        locs = getFinalLocs(numSteps, numTrials, dClass)
        xVals, yVals = [], []
        for loc in locs:
            xVals.append(loc.getX())
            yVals.append(loc.getY())
        xVals = np.array(xVals)
        yVals = np.array(yVals)
        meanX = sum(abs(xVals))/len(xVals)
        meanY = sum(abs(yVals))/len(yVals)

        curStyle = styleChoice.nextStyle()
        plt.plot(xVals, yVals, curStyle,
                   label=dClass.__name__ + \
                         ' mean abs dist = <'
                         + str(meanX) + ', ' + str(meanY) + '>')
    plt.ylim(-1000, 1000)
    plt.xlim(-1000, 1000)
    plt.show()

random.seed(0)
#drunkTest([10, 100, 1000, 10000], 100, UsualDrunk)
#simAll((UsualDrunk, ColdDrunk), (1, 10, 100, 1000, 10000), 100)

numSteps = (10, 100, 1000, 10000)
simAll((UsualDrunk, ColdDrunk), numSteps, 100)
plotLocs((UsualDrunk, ColdDrunk), 10000, 100)


import matplotlib.pyplot as plt

# Definindo os limites dos eixos
dim = [100, 100]
plt.xlim(-dim[0], dim[0])
plt.ylim(-dim[1], dim[1])
# Mantendo a proporção dos eixos
plt.gca().set_aspect('equal', adjustable='box')

plt.axhline(0, color='black', linestyle='--')
plt.axvline(0, color='black', linestyle='--')

centro_pos = (dim[0] / 2, dim[1] / 2)

pts_x = [centro_pos[0] - 25,
         centro_pos[0] + 30,
         centro_pos[0] - 25]

pts_y = [centro_pos[1] - 25,
         centro_pos[1] - 25,
         centro_pos[1] + 10]

triangulo = plt.Polygon([[pts_x[0], pts_y[0]],
                           [pts_x[1], pts_y[1]],
                           [pts_x[2], pts_y[2]]],
                          closed=True,
                          fill = True,
                          edgecolor='blue',
                          color='blue',
                          label = "Original")

plt.gca().add_patch(triangulo)

# espelhado no eixo X
xRX = []
for x in pts_x:
  x = -x
  xRX.append(x)

triangulo_eX = plt.Polygon([[xRX[0], pts_y[0]],
                           [xRX[1], pts_y[1]],
                           [xRX[2], pts_y[2]]],
                          closed=True,
                          fill = True,
                          edgecolor='red',
                          color='red',
                          label = "Espelhado no Eixo X")
plt.gca().add_patch(triangulo_eX)

# espelhado no eixo Y
yRY = []
for y in pts_y:
  y = -y
  yRY.append(y)

triangulo_eY = plt.Polygon([[pts_x[0], yRY[0]],
                            [pts_x[1], yRY[1]],
                            [pts_x[2], yRY[2]]],
                          closed=True,
                          fill = True,
                          color = 'green',
                          edgecolor='green',
                          label = "Espelhado no Eixo Y")

plt.gca().add_patch(triangulo_eY)

plt.legend()
plt.show()


import torch
from torch.nn import functional as F
import torch.nn as nn
import numpy as np

# Reconstruction + KL divergence losses summed over all elements and batch
# note the change of var order!
def loss_bce_kld(x, recon_x, mu, logvar):
    BCE = F.binary_cross_entropy(recon_x.view(-1, 1), x.view(-1, 1), reduction='sum')

    # see Appendix B from VAE paper:
    # Kingma and Welling. Auto-Encoding Variational Bayes. ICLR, 2014
    # https://arxiv.org/abs/1312.6114
    # 0.5 * sum(1 + log(sigma^2) - mu^2 - sigma^2)
    KLD = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp())

    return BCE + KLD, KLD, BCE

# Note switch x, recon_x
def loss_rho_bce_kld(x, recon_x, mu, rho, logs, z_dim):

    BCE = F.binary_cross_entropy(recon_x.view(-1, 1), x.view(-1, 1), reduction='sum')
    ##
    KLD = 0.5 * (torch.sum(mu.pow(2)) + - z_dim * logs - (z_dim - 1) * torch.log(1 - rho**2) + z_dim * (logs.exp() - 1))
    KLD = torch.mean(KLD)

    return BCE + KLD, KLD, BCE

def reconstruction_example(model, use_rho, data_loader, img_shape, num_class, use_cuda):

    model.eval()
    img_shape = img_shape[1:]

    x, y = next(iter(data_loader))
    x = to_cuda(x) if use_cuda else x
    if use_rho:
        x_hat, _, _, _ = model(x)
    else:
        x_hat, _, _ = model(x)

    x = x[:num_class].cpu().view(num_class * img_shape[0], img_shape[1])
    x_hat = x_hat[:num_class].cpu().view(num_class * img_shape[0], img_shape[1])
    comparison = torch.cat((x, x_hat), 1).view(num_class * img_shape[0], 2 * img_shape[1])
    return comparison

def generation_example(model, latent_size, data_loader, img_shape, num_class, use_cuda):
    img_shape = img_shape[1:]

    draw = randn((num_class, latent_size), use_cuda)
    sample = model.decode(draw).cpu().view(num_class, 1, img_shape[0], img_shape[1])

    return sample

def to_cuda(tensor):
    if isinstance(tensor, torch.Tensor):
        tensor = tensor.cuda()
    return tensor

def init_weights(module):
    for m in module.modules():
        if isinstance(m, nn.Linear) or isinstance(m, nn.Conv2d):
            nn.init.xavier_uniform_(m.weight.data)
            if hasattr(m, 'bias') and m.bias is not None:
                nn.init.constant_(m.bias, 0.0)
        elif isinstance(m, nn.Sequential):
            for sub_mod in m:
                init_weights(sub_mod)

# https://github.com/jramapuram/helpers/utils.py
def randn(shape, cuda, mean=0, sigma=1, dtype='float32'):
    shape = list(shape) if isinstance(shape, tuple) else shape
    return type_map[dtype](cuda)(*shape).normal_(mean, sigma)

def type_tdouble(use_cuda=False):
    return torch.cuda.DoubleTensor if use_cuda else torch.DoubleTensor

def type_tfloat(use_cuda=False):
    return torch.cuda.FloatTensor if use_cuda else torch.FloatTensor

def type_tint(use_cuda=False):
    return torch.cuda.IntTensor if use_cuda else torch.IntTensor

def type_tlong(use_cuda=False):
    return torch.cuda.LongTensor if use_cuda else torch.LongTensor

def conv_size(H_in, k_size, stride, padd, dil=1):
    H_out = np.floor((H_in + 2 * padd - dil * (k_size - 1) - 1) / stride + 1)
    return np.int(H_out)

def save_checkpoint(state, filename):
    torch.save(state, filename)

# https://github.com/jramapuram/helpers/utils.py
type_map = {
    'float32': type_tfloat,
    'float64': type_tdouble,
    'double': type_tdouble,
    'int32': type_tint,
    'int64': type_tlong
}


from selenium import webdriver
from bs4 import BeautifulSoup
import time
import pandas

# time.sleep(5) # Let the user actually see something!
# driver.quit()

class Crawler:
    def __init__(self):
        self.driver = webdriver.Chrome()
        self.link = []
        time.sleep(2)

    def browse(self):
        self.driver.get('https://google.com')
        click = self.driver.find_element_by_name('q')
        click.send_keys('allinurl:  site:.go.id')
        click.submit()
        time.sleep(2)

    def collect(self):
        content = self.driver.page_source
        get_content = BeautifulSoup(content)
        for a in get_content.find_all('a', href=True, attrs={'class': 'iUh30 tjvcx'}):
            link = a.find('cite', attrs={'class': 'iUh30 tjvcx'})
            self.link.append(link.text)
        time.sleep(2)
        self.driver.quit()

    def save(self):
        db = pandas.DataFrame({'Link': self.link})
        db.to_csv('link.csv', index=False, encoding='utf-8')

bot = Crawler()
bot.browse()
bot.collect()
bot.save()


from __future__ import unicode_literals

import tempfile
import shutil
from io import StringIO
import os

from django.test import TestCase
import mock

from archives.models import ArchiveArtifact
from archives.transports import LocalTransport, SshTransport
from jenkins.tests.factories import ArtifactFactory, BuildFactory
from projects.helpers import build_project
from projects.models import ProjectDependency, ProjectBuildDependency
from projects.tests.factories import ProjectFactory, DependencyFactory
from .factories import ArchiveFactory

class LocalTransportTest(TestCase):

    def setUp(self):
        self.basedir = tempfile.mkdtemp()
        self.archive = ArchiveFactory.create(
            transport="local", basedir=self.basedir)

    def tearDown(self):
        shutil.rmtree(self.basedir)

    def test_archive_file(self):
        """
        LocalTransport should copy the file from the supplied fileobj to a
        local path.
        """
        transport = LocalTransport(self.archive)
        fakefile = StringIO(u"This is the artifact")

        size = transport.archive_file(fakefile, "/temp/temp.gz")

        self.assertEqual(20, size)
        filename = os.path.join(self.basedir, "temp/temp.gz")
        self.assertEqual(file(filename).read(), "This is the artifact")

    def test_archive_from_url(self):
        """
        archive_from_url takes a valid URL and opens the file and then passes
        it to archive_file.
        """
        fakefile = StringIO(u"Entirely new artifact")

        mock_request = mock.Mock()
        with mock.patch("archives.transports.urllib2") as urllib2_mock:
            urllib2_mock.Request.return_value = mock_request
            transport = LocalTransport(self.archive)
            urllib2_mock.urlopen.return_value = fakefile
            transport.archive_url(
                "http://example.com/testing", "/temp/temp.gz",
                "username", "password")
        urllib2_mock.urlopen.assert_called_once_with(mock_request)
        urllib2_mock.Request.assert_called_once_with(
            "http://example.com/testing")
        mock_request.assert_has_calls(
            mock.call.add_header(
                "Authorization", "Basic dXNlcm5hbWU6cGFzc3dvcmQ="))
        filename = os.path.join(self.basedir, "temp/temp.gz")
        self.assertEqual(file(filename).read(), "Entirely new artifact")

    def test_start(self):
        """
        LocalTransport.start should ensure that the basedir exists.
        """
        dirname = os.path.join(self.basedir, "temp")
        self.archive.basedir = dirname
        transport = LocalTransport(self.archive)
        transport.start()

        self.assertTrue(os.path.exists(dirname))

    def test_link_filename_to_filename(self):
        """
        LocalTransport.link_filename_to_filename should hardlink the source to
        the destination.
        """
        transport = LocalTransport(self.archive)
        fakefile = StringIO("This is the artifact")

        transport.archive_file(fakefile, "/temp/temp.gz")
        transport.link_filename_to_filename("/temp/temp.gz", "/temp/temp1.gz")

        filename1 = os.path.join(self.basedir, "temp/temp.gz")
        # Test that the number of links to this file is 2
        # (the original, and the newly created hardlink)
        self.assertEqual(2, os.stat(filename1).st_nlink)
        filename2 = os.path.join(self.basedir, "temp/temp1.gz")
        self.assertEqual(file(filename2).read(), "This is the artifact")

    def test_link_artifact_to_current(self):
        """
        Create a link in the parent directory for this artifact,
        pointing to the directory of the artifact, with the name "current".
        """
        transport = LocalTransport(self.archive)
        fakefile = StringIO("This is the artifact")

        transport.archive_file(fakefile, "/temp/temp.gz")
        current_dir = transport.link_to_current("/temp/temp.gz")

        self.assertEqual(os.path.join(self.basedir, "current"), current_dir)
        self.assertTrue(
            os.path.islink(current_dir), "current is not a symlink")
        self.assertEqual(
            os.path.dirname(transport.get_relative_filename("/temp/temp.gz")),
            os.path.realpath(current_dir))

    def test_link_artifact_to_current_replaces_existing_current(self):
        """
        If we run link_artifact_to_current with a new path, it should ensure
        that the "current" directory is the new one.
        """
        transport = LocalTransport(self.archive)
        fakefile = StringIO("This is the artifact")

        transport.archive_file(fakefile, "/temp1/temp.gz")
        transport.link_to_current("/temp1/temp.gz")

        fakefile = StringIO("Another artifact")
        transport.archive_file(fakefile, "/temp2/temp.gz")
        current_dir = transport.link_to_current("/temp2/temp.gz")

        self.assertEqual(
            os.path.dirname(transport.get_relative_filename("/temp2/temp.gz")),
            os.path.realpath(current_dir))

class SshTransportTest(TestCase):

    def setUp(self):
        self.archive = ArchiveFactory.create(
            transport="ssh", basedir="/var/tmp")

    def test_get_ssh_clients(self):
        """
        _get_ssh_clients should return an SSHClient and SFTPClient configured
        to talk to the archive"s credentials.
        """
        with mock.patch.object(self.archive.ssh_credentials, "get_pkey", return_value="KEY"):  # noqa
            with mock.patch("archives.transports.SSHClient") as mock_client:
                mock_client.return_value.get_transport.return_value = "MockTransport"  # noqa
                with mock.patch("archives.transports.SFTPClient") as mock_sftp:
                    with mock.patch("archives.transports.WarningPolicy") as mock_hostpolicy:  # noqa
                        mock_hostpolicy.return_value = "MockWarningPolicy"
                        transport = SshTransport(self.archive)
                        transport._get_ssh_clients()

        mock_client.return_value.assert_has_calls([
            mock.call.set_missing_host_key_policy("MockWarningPolicy"),
            mock.call.connect(
                "archive.example.com", username="testing", pkey="KEY"),
            mock.call.get_transport()])
        mock_sftp.from_transport.assert_called_once_with("MockTransport")

    def test_archive_file(self):
        """
        archive_file should ensure that there's a directory relative to the
        base to hold the file, and then stream the file to the remote server.
        """
        mock_ssh = mock.Mock()
        mock_stdout = mock.Mock()
        mock_ssh.exec_command.return_value = None, mock_stdout, None
        mock_sftp = mock.Mock()
        fakefile = StringIO(u"This is the artifact")

        transport = SshTransport(self.archive)
        with mock.patch.object(
                transport, "_get_ssh_clients",
                return_value=(mock_ssh, mock_sftp)):
            transport.start()
            transport.archive_file(fakefile, "/temp/temp.gz")
        mock_ssh.exec_command.assert_called_once_with(
            "mkdir -p `dirname /var/tmp/temp/temp.gz`")

        mock_sftp.stream_file_to_remote.assert_called_once_with(
            fakefile, "/var/tmp/temp/temp.gz")

        mock_ssh.close.assert_called_once()

    def test_generate_checksums(self):
        """
        generate_checksums should send commands to the ssh client
        to generate an sha256sum for the passed in archived artifact.
        """
        # a project with a build and an archived artifact
        project = ProjectFactory.create()
        dependency = DependencyFactory.create()
        ProjectDependency.objects.create(
            project=project, dependency=dependency)
        projectbuild = build_project(project, queue_build=False)
        build = BuildFactory.create(
            job=dependency.job, build_id=projectbuild.build_key)
        projectbuild_dependency = ProjectBuildDependency.objects.create(
            build=build, projectbuild=projectbuild, dependency=dependency)
        artifact = ArtifactFactory.create(
            build=build, filename="artifact_filename")
        archived_artifact = ArchiveArtifact.objects.create(
            build=build, archive=self.archive, artifact=artifact,
            archived_path="/srv/builds/200101.01/artifact_filename",
            projectbuild_dependency=projectbuild_dependency)

        transport = SshTransport(self.archive)

        with mock.patch.object(transport, "_run_command") as mock_run:
            transport.generate_checksums(archived_artifact)

        mock_run.assert_called_once_with(
            "cd `dirname /var/tmp/srv/builds/200101.01/artifact_filename` "
            "&& sha256sum artifact_filename >> SHA256SUMS")

    def test_link_filename_to_filename(self):
        """
        archive_file should ensure that there's a directory relative to the
        base to hold the file, and then stream the file to the remote server.
        """
        mock_ssh = mock.Mock()
        mock_stdout = mock.Mock()
        mock_ssh.exec_command.return_value = None, mock_stdout, None
        mock_sftp = mock.Mock()

        transport = SshTransport(self.archive)
        with mock.patch.object(
                transport, "_get_ssh_clients",
                return_value=(mock_ssh, mock_sftp)):
            transport.start()
            transport.link_filename_to_filename(
                "/temp/temp.gz", "/temp2/temp.gz")
        mock_ssh.exec_command.assert_has_calls(
            [mock.call("mkdir -p `dirname /var/tmp/temp2/temp.gz`"),
             mock.call('ln "/var/tmp/temp/temp.gz" "/var/tmp/temp2/temp.gz"')])

        mock_ssh.close.assert_called_once()

    def test_link_artifact_to_current(self):
        """
        Create a link in the parent directory for this artifact,
        pointing to the directory of the artifact, with the name "current".
        """
        mock_ssh = mock.Mock()
        mock_stdout = mock.Mock()
        mock_ssh.exec_command.return_value = None, mock_stdout, None
        mock_sftp = mock.Mock()

        transport = SshTransport(self.archive)
        with mock.patch.object(
                transport, "_get_ssh_clients",
                return_value=(mock_ssh, mock_sftp)):
            transport.start()
            current_dir = transport.link_to_current("/project/1234/temp.gz")
        mock_ssh.exec_command.assert_has_calls(
            [mock.call('cd "/var/tmp/project" && rm -f current '
                       '&& ln -sf "1234" current')])
        self.assertEqual("/var/tmp/project/current", current_dir)
        mock_ssh.close.assert_called_once()


import logging

import boto3
from box import Box
from crhelper import CfnResource
from schema import Optional

from codesmith.common import naming
from codesmith.common.calc import calculator, SSM_PARAMETER_DESCRIPTION
from codesmith.common.cfn import resource_properties
from codesmith.common.schema import non_empty_string, tolerant_schema
from codesmith.common.ssm import silent_delete_parameter_from_event, put_string_parameter

helper = CfnResource()
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

ssm = boto3.client('ssm')
expression_parser = calculator(1)

properties_schema = tolerant_schema({
    'SequenceName': non_empty_string,
    Optional('Expression'): non_empty_string
})

def validate_properties(properties):
    p = properties_schema.validate(properties)
    if 'expression' not in p:
        p['expression'] = 'x'
    expression_parser.parse(p['expression'])
    return Box(p, camel_killer_box=True)

@helper.create
@helper.update
def create(event, _):
    properties = validate_properties(resource_properties(event))
    return put_sequence_parameter(properties)

def put_sequence_parameter(properties):
    parameter_name = naming.sequence_parameter_name(properties.sequence_name)
    put_string_parameter(ssm, parameter_name,
                         value=properties.expression,
                         description=SSM_PARAMETER_DESCRIPTION)
    return parameter_name

@helper.delete
def delete(event, _):
    return silent_delete_parameter_from_event(ssm, event)

def handler(event, context):
    logger.info('event: %s', event)
    helper(event, context)


import streamlit as st
import constants as c
import pandas as pd
from utils import (display_all_sorting_algorithms,
                   display_algorithm_type, get_object_from_factory,
                   display_bar_chart)
import numpy as np

def main():

    hide_github_icon = """
            #GithubIcon {
            visibility: hidden;
        }
    """
    # Page layout configurations
    st.set_page_config(page_title="Algorithm Visualizer", layout="wide", initial_sidebar_state="expanded")

    st.title("🔍 Algorithm Visualizer")

    st.write("""
    Welcome to the **Algorithm Visualizer**! This tool helps you visualize various sorting algorithms in action.
    Use the dropdown below to select the type of sorting algorithm you'd like to visualize.
    """)

    # Create a sidebar for controls and info
    with st.sidebar:
        st.header("Menu")
        algorithm_type = st.selectbox("Algorithm Type:", display_algorithm_type())
        sorting_algorithm = None
        if algorithm_type == c.SORTING_ALGORITHM:
            sorting_algorithm = st.selectbox("Sorting Algorithm:", display_all_sorting_algorithms())
            array_len = st.slider("Array Size:", 5, 100, 20)

    # Main content
    with st.container():
        if not sorting_algorithm:  # If no sorting algorithm is selected
            st.header("Complexities of Common Sorting Algorithms")
            st.write("Below are the complexities of some common sorting algorithms for your reference.")
            df = pd.DataFrame.from_dict(c.ALGORITHM_COMPLEXITIES, orient='index')
            st.table(df)

        elif algorithm_type == c.SORTING_ALGORITHM:
            st.subheader(f"Visualizing: {sorting_algorithm}")
            array = list(np.random.randint(0, 250, array_len))

            if sorting_algorithm:
                display_bar_chart(array, is_initial=True)

                if array:
                    col1, col2, col3 = st.columns([1,1,1])

                    with col2:
                        sort_pressed = st.button("Sort")

                if sort_pressed:
                    algorithm_object = get_object_from_factory(sorting_algorithm, c.SORTING_ALGORITHM)
                    algorithm_object.sort(array, callback=algorithm_object.display_bars)
                    st.balloons()

        st.info("🔜 This site will soon move to [DSA Ground](https://dsaground.streamlit.app/)")

if __name__ == "__main__":
    main()


import re
import string
import ipaddress

from . import utils

class RequestProtocol(object):
    METHOD_GET = 'GET'
    METHOD_OPTIONS = 'OPTIONS'
    METHOD_HEAD = 'HEAD'
    METHOD_POST = 'POST'
    METHOD_PUT = 'PUT'
    METHOD_DELETE = 'DELETE'
    METHOD_TRACE = 'TRACE'
    METHOD_CONNECT = 'CONNECT'

    HTTP_VERSION = '1.1'

    CRLF = '\r\n'
    SP = ' '

    REQUEST_LINE = '{method}{sp}{request_uri}{sp}HTTP/{http_version}{crlf}'

    @classmethod
    def path(cls, uri):
        path = uri.path
        if uri.query:
            path = '{}?{}'.format(path, uri.query)
        return path

    @classmethod
    def header_fields(cls, headers):
        return '\r\n'.join(
            '{}: {}'.format(h, v) for h, v in headers.items())

    @classmethod
    def request_line(cls, method, request_uri):
        return cls.REQUEST_LINE.format(
            method=method,
            sp=cls.SP,
            request_uri=request_uri,
            http_version=cls.HTTP_VERSION,
            crlf=cls.CRLF)

class ResponseProtocol(object):
    COLON = ':'
    HTTP = 'HTTP/'

    REGEX_CHARSET = re.compile(r';\s*charset=([^;]*)', re.I)
    REGEX_TOKEN = re.compile(
        r'([a-zA-Z][a-zA-Z_-]*)\s*(?:=(?:"([^"]*)"|'
        r'([^ \t",;]*)))?')

    @classmethod
    def parse_status(cls, status):
        if status.startswith(cls.HTTP):
            http_version, status_code, status_text = status.split(None, 2)
            status = '{} {}'.format(
                utils.smart_text(status_code),
                utils.smart_text(status_text))
        return status

    @classmethod
    def parse_status_code(cls, status):
        return int(status.split()[0])

    @classmethod
    def parse_charset(cls, header, charset):
        match = cls.REGEX_CHARSET.search(utils.smart_text(header))
        if match:
            charset = match.group(1)
        return charset

    @classmethod
    def parse_cache_control(cls, header):
        header = utils.smart_text(header)
        cache = {}
        for match in cls.REGEX_TOKEN.finditer(header):
            name = match.group(1)
            value = match.group(2) or match.group(3) or None
            if value and value.isdigit():
                value = int(value)
            cache[name] = value

        cache_control = {}
        for n in [
            'public',
            'no-store',
            'no-transform',
            'must-revalidate',
            'proxy-revalidate',
        ]:
            if n not in cache:
                continue
            cache_control[n] = None

        for n, v in cache.items():
            if n not in [
                'private',
                'no-cache',
                'max-age',
                's-maxage',
                'stale-while-revalidate',
                'stale-if-error',
            ]:
                continue
            cache_control[n] = v
        return cache_control

class URIProtocol(object):
    ALPHA = string.ascii_letters
    DIGIT = string.digits
    UNRESERVED = ALPHA + DIGIT + '-' '.' '_' '~'

    GEN_DELIMS = ':' '/' '?' '#' '[' ']' '@'
    SUB_DELIMS = '!' '$' '&' '\'' '(' ')' '*' '+' ',' ';' '='
    RESERVED = GEN_DELIMS + SUB_DELIMS

    PCT_ENCODED = '%' + string.hexdigits
    PCHAR = UNRESERVED + PCT_ENCODED + SUB_DELIMS + ':' + '@'

    # Parts
    SCHEME = ALPHA + DIGIT + '+' + '-' + '.'
    QUERY = PCHAR + '/' '?'
    FRAGMENT = PCHAR + '/' '?'

    # Authority
    USERINFO = UNRESERVED + PCT_ENCODED + SUB_DELIMS + ':'
    PORT = DIGIT

    # Host
    REG_NAME = UNRESERVED + PCT_ENCODED + SUB_DELIMS

    # Path
    SEGMENT = PCHAR
    SEGMENT_NZ_NC = UNRESERVED + PCT_ENCODED + SUB_DELIMS + '@'

    @classmethod
    def strip_scheme(cls, uri):
        if ':' not in uri:
            return None, uri

        scheme, hier_part = uri.split(':', 1)
        if scheme[0] not in cls.ALPHA:
            raise exc.SchemeException(uri)

        if any(c not in cls.SCHEME for c in scheme[1:]):
            raise exc.SchemeException(scheme)

        return scheme.lower(), hier_part

    @classmethod
    def strip_fragment(cls, hier_part):
        if '#' in hier_part:
            hier_part, fragment = \
                hier_part.rsplit('#', 1)
            if any(c not in cls.FRAGMENT for c in fragment):
                raise exc.FragmentException(fragment)
        else:
            fragment = None
        return fragment, hier_part

    @classmethod
    def strip_query(cls, hier_part):
        if '?' in hier_part:
            hier_part, query = hier_part.rsplit('?', 1)
            if any(c not in cls.QUERY for c in query):
                raise exc.QueryException(query)
        else:
            query = None
        return query, hier_part

    @classmethod
    def strip_authority(cls, hier_part):
        if hier_part.startswith('//'):
            hier_part = hier_part[2:]
        if '/' in hier_part:
            authority, hier_part = hier_part.split('/', 1)
            hier_part = '/{}'.format(hier_part)
        else:
            authority = hier_part
            hier_part = ''
        return authority, hier_part

    @classmethod
    def strip_userinfo(cls, authority):
        if '@' in authority:
            userinfo, authority = authority.split('@', 1)
            if any(c not in cls.USERINFO for c in userinfo):
                raise exc.UserInfoException(userinfo)
            if not userinfo:
                userinfo = None
        else:
            userinfo = None
        return userinfo, authority

    @classmethod
    def strip_port(cls, authority):
        if ':' in authority:
            authority, port = authority.rsplit(':', 1)
        else:
            return None, authority
        if port.isdigit():
            port = int(port)
        else:
            raise exc.PortException(port)
        return port, authority

    @classmethod
    def verify_reg_name(cls, host):
        return all(c in cls.REG_NAME for c in host)

    @classmethod
    def verify_ipv4_address(cls, host):
        if '.' not in host:
            return False

        try:
            ipaddress.IPv4Address(host)
        except ipaddress.AddressValueError:
            return False
        else:
            return True

    @classmethod
    def verify_ipv6_address(cls, host):
        if ':' not in host:
            return False

        try:
            ipaddress.IPv6Address(host)
        except ipaddress.AddressValueError:
            return False
        else:
            return True

    @classmethod
    def verify_path_abempty(cls, path):
        if not path:
            return True

        if not path.startswith('/'):
            return False

        segments = path.split('/')
        for segment in segments:
            if any(c not in cls.SEGMENT for c in segment):
                return False
        else:
            return True

    @classmethod
    def verify_path_absolute(cls, path):
        if not path.startswith('/'):
            return False

        segments = path.split('/')[1:]
        segment = segments[0]
        if not segment or any(c not in cls.SEGMENT for c in segment):
            return False

        for segment in segments[1:]:
            if any(c not in cls.SEGMENT for c in segment):
                return False
        else:
            return True

    @classmethod
    def verify_path_noscheme(cls, path):
        if path.startswith('/'):
            return False

        segments = path.split('/')
        segment = segments[0]
        if not segment or any(c not in cls.SEGMENT_NZ_NC for c in segment):
            return False

        for segment in segments[1:]:
            if any(c not in cls.SEGMENT for c in segment):
                return False
        else:
            return True

    @classmethod
    def verify_path_rootless(cls, path):
        if path.startswith('/'):
            return False

        segments = path.split('/')
        segment = segments[0]
        if not segment or any(c not in cls.SEGMENT for c in segment):
            return False

        for segment in segments[1:]:
            if any(c not in cls.SEGMENT for c in segment):
                return False
        else:
            return True

    @classmethod
    def verify_path_empty(cls, path):
        return bool(path)


import factory
import random

from django.core.management.base import BaseCommand
from django.conf import settings

import donate.core.factory as core_factory
from donate.utility.faker.helpers import reseed

from wagtail_factories import ImageFactory

class Command(BaseCommand):
    help = 'Generate fake data for local development and testing purposes' \
           'and load it into the database'

    def add_arguments(self, parser):
        parser.add_argument(
            '--seed',
            action='store',
            dest='seed',
            help='A seed value to pass to Faker before generating data',
        )

    def handle(self, *args, **options):

        faker = factory.faker.Faker._get_faker(locale='en-US')

        # Seed Faker with the provided seed value or a pseudorandom int between 0 and five million
        if options['seed']:
            seed = options['seed']
        elif settings.RANDOM_SEED is not None:
            seed = settings.RANDOM_SEED
        else:
            seed = random.randint(0, 5000000)

        self.stdout.write(f'Seeding random numbers with: {seed}')

        reseed(seed)

        self.stdout.write('Generating Images')
        for i in range(20):
            ImageFactory.create(
                file__width=1080,
                file__height=720,
                file__color=faker.safe_color_name()
            )

        factories = [
            core_factory,
        ]
        for app_factory in factories:
            app_factory.generate(seed)

        self.stdout.write(self.style.SUCCESS('Done!'))


import collections;
import json;
import fcntl;

# Config = collections.namedtuple('Config', 'blog_name blog_subname nav_list');

def load_config(config_file):
    fin = open(config_file, 'r');
    fcntl.flock(fin, fcntl.LOCK_SH);
    config = json.loads(fin.read());
    fcntl.flock(fin, fcntl.LOCK_UN);
    fin.close();

    return config;

def dump_config(config, config_file):
    s = json.dumps(config, indent = 4);
    fout = open(config_file, 'w');
    fcntl.flock(fout, fcntl.LOCK_EX);
    fout.write(s);
    fcntl.flock(fout, fcntl.LOCK_UN);
    fout.close();

# print(load_config('../data/config.cnf'));
# for test


import os
import requests
from datetime import datetime, timedelta
from pprint import pprint

TEQUILA_API_KEY = os.environ["TEQUILA_API_KEY"]
TEQUILA_ENDPOINT = "https://tequila-api.kiwi.com"

class FlightSearch:

    def get_iata_code(self, term):
        params = {
            "term": term,
            "location_types": "airport"
        }
        headers = {
            "apikey": TEQUILA_API_KEY,
            "accept": "application / json"
        }
        endpoint = f"{TEQUILA_ENDPOINT}/locations/query"
        r = requests.get(endpoint, params=params, headers=headers)
        r = r.json()["locations"][0]
        return r["city"]["code"]

    def get_cheapest_flight(self, my_place, destination, max_stay=(6 * 30)):
        tomorrow = datetime.now() + timedelta(days=1)
        params = {
            "fly_from": my_place,
            "fly_to": destination['iataCode'],
            "date_from": tomorrow.strftime("%d/%m/%Y"),
            "date_to": (tomorrow + timedelta(days=max_stay)
                        ).strftime("%d/%m/%Y"),
            "nights_in_dst_from": 7,
            "nights_in_dst_to": 28,
            "flight_type": "round",
            "one_for_city": 1,
            "max_stopovers": 0,
            "curr": "GBP",
        }
        headers = {
            "apikey": TEQUILA_API_KEY,
            "accept": "application / json"
        }
        endpoint = f"{TEQUILA_ENDPOINT}/v2/search"
        r = requests.get(endpoint, params=params, headers=headers)
        try:
            data = r.json()["data"][0]
        except IndexError:
            print(f"No flights found for {destination}.")
            try:
                params["max_stopovers"] = 1
                r = requests.get(endpoint, params=params, headers=headers)
                data = r.json()["data"][0]
                pprint(data)
            except IndexError:
                return
        return data


import hashlib
import os
import mimetypes
import subprocess
import json
import chardet
import warnings
import docx2txt
import xlrd
from pdfminer.pdfparser import PDFParser, PDFSyntaxError
from pdfminer.pdfdocument import PDFDocument
from pdfminer.pdfpage import PDFPage
from pdfminer.pdfinterp import PDFResourceManager, PDFPageInterpreter
from pdfminer.layout import LAParams, LTTextBox, LTTextLine
from pdfminer.converter import PDFPageAggregator
import html2text
from ebooklib import epub
import ebooklib
from PIL import Image
from fontTools.ttLib import TTFont, TTLibError
import six
from six.moves import xrange

class MimeGuesser:
    def guess_mime(self, full_path):
        raise NotImplementedError()

class ContentMimeGuesser(MimeGuesser):

    def __init__(self):
        import magic
        self.libmagic = magic.Magic(mime=True)

    def guess_mime(self, full_path):
        try:
            return self.libmagic.from_file(full_path)
        except FileNotFoundError:
            return None

class ExtensionMimeGuesser(MimeGuesser):
    def guess_mime(self, full_path):
        return mimetypes.guess_type(full_path, strict=False)[0]

class FileParser:
    mime_types = []
    is_default = False

    def parse(self, full_path: str):
        raise NotImplemented

class FileCheckSumCalculator:
    def checksum(self, path: str) -> str:
        """
        Calculate the checksum of a file
        :param path: path of the file
        :return: checksum
        """
        raise NotImplementedError()

class Md5CheckSumCalculator(FileCheckSumCalculator):
    def __init__(self):
        self.name = "md5"

    def checksum(self, path: str) -> str:
        """
        Calculate the md5 checksum of a file
        :param path: path of the file
        :return: md5 checksum
        """
        result = hashlib.md5()

        with open(path, "rb") as f:
            for block in iter(lambda: f.read(65536), b""):
                result.update(block)

        return result.hexdigest().upper()

class Sha1CheckSumCalculator(FileCheckSumCalculator):
    def __init__(self):
        self.name = "sha1"

    def checksum(self, path: str) -> str:
        """
        Calculate the sha1 checksum of a file
        :param path: path of the file
        :return: sha1 checksum
        """
        result = hashlib.sha1()

        with open(path, "rb") as f:
            for block in iter(lambda: f.read(65536), b""):
                result.update(block)

        return result.hexdigest().upper()

class Sha256CheckSumCalculator(FileCheckSumCalculator):
    def __init__(self):
        self.name = "sha256"

    def checksum(self, path: str) -> str:
        """
        Calculate the sha256 checksum of a file
        :param path: path of the file
        :return: sha256 checksum
        """
        result = hashlib.sha256()

        with open(path, "rb") as f:
            for block in iter(lambda: f.read(65536), b""):
                result.update(block)

        return result.hexdigest().upper()

class GenericFileParser(FileParser):
    mime_types = []
    is_default = True

    def __init__(self, checksum_calculators: list, root_dir: str):
        self.checksum_calculators = checksum_calculators
        self.root_dir = root_dir
        self.root_dir_len = len(root_dir)+1

    def parse(self, full_path: str) -> dict:
        """
        Parse a generic file
        :param full_path: path of the file to parse
        :return: dict information about the file
        """

        info = dict()

        file_stat = os.stat(full_path)
        path, name = os.path.split(full_path)
        name, extension = os.path.splitext(name)

        info["size"] = file_stat.st_size
        info["path"] = os.path.relpath(path, self.root_dir)
        info["name"] = name
        info["extension"] = extension[1:]
        info["mtime"] = file_stat.st_mtime

        # TODO: calculate all checksums at once
        for calculator in self.checksum_calculators:
            info[calculator.name] = calculator.checksum(full_path)

        return info

class MediaFileParser(GenericFileParser):
    is_default = False
    relevant_properties = ["bit_rate", "nb_streams", "duration", "format_name", "format_long_name"]

    def __init__(self, checksum_calculators: list, root_dir):
        super().__init__(checksum_calculators, root_dir)

        self.mime_types = [
            "video/3gpp", "video/mp4", "video/mpeg", "video/ogg", "video/quicktime",
            "video/webm", "video/x-flv", "video/x-mng", "video/x-ms-asf",
            "video/x-ms-wmv", "video/x-msvideo", "audio/basic", "auido/L24",
            "audio/mid", "audio/mpeg", "audio/mp4", "audio/x-aiff",
            "audio/ogg", "audio/vorbis" "audio/x-realaudio", "audio/x-wav",
            "audio/flac", "audio/x-monkeys-audio", "audio/wav", "audio/wave",
            "audio/x-wav", "audio/x-ms-wma", "audio/x-flac",
        ]

    def parse(self, full_path: str):
        info = super().parse(full_path)

        p = subprocess.Popen(["ffprobe", "-v", "quiet", "-print_format", "json=c=1", "-show_format", full_path],
                             stdout=subprocess.PIPE)
        out, err = p.communicate()

        try:
            metadata = json.loads(out.decode("utf-8"))

            if "format" in metadata:

                if "duration" in metadata["format"]:
                    info["duration"] = float(metadata["format"]["duration"])

                if "format_long_name" in metadata["format"]:
                    info["format_long_name"] = metadata["format"]["format_long_name"]

                if "tags" in metadata["format"]:
                    if "genre" in metadata["format"]["tags"]:
                        info["genre"] = metadata["format"]["tags"]["genre"]
                    if "title" in metadata["format"]["tags"]:
                        info["title"] = metadata["format"]["tags"]["title"]
                    if "album" in metadata["format"]["tags"]:
                        info["album"] = metadata["format"]["tags"]["album"]
                    if "album_artist" in metadata["format"]["tags"]:
                        info["album_artist"] = metadata["format"]["tags"]["album_artist"]

        except json.decoder.JSONDecodeError:
            print("json decode error:" + full_path)
            pass

        return info

class PictureFileParser(GenericFileParser):
    is_default = False

    def __init__(self, checksum_calculators: list, root_dir):
        super().__init__(checksum_calculators, root_dir)

        self.mime_types = [
            "image/bmp", "image/cgm", "image/cis-cod", "image/g3fax", "image/gif",
            "image/ief", "image/jpeg", "image/ktx", "image/pipeg", "image/pjpeg",
            "image/png", "image/prs.btif", "image/svg+xml", "image/tiff",
            "image/vnd.adobe.photoshop", "image/vnd.dece.graphic", "image/vnd.djvu",
            "image/vnd.dvb.subtitle", "image/vnd.dwg", "image/vnd.dxf",
            "image/vnd.fastbidsheet", "image/vnd.fpx", "image/vnd.fst",
            "image/vnd.fujixerox.edmics-mmr", "image/vnd.fujixerox.edmics-rlc",
            "image/vnd.ms-modi", "image/vnd.net-fpx", "image/vnd.wap.wbmp",
            "image/vnd.xiff", "image/webp", "image/x-citrix-jpeg", "image/x-citrix-png",
            "image/x-cmu-raster", "image/x-cmx", "image/x-icon",
            "image/x-pcx", "image/x-pict", "image/x-png", "image/x-portable-bitmap",
            "image/x-portable-graymap", "image/x-portable-pixmap",
            "image/x-rgb", "image/x-xbitmap", "image/x-xpixmap", "image/x-xwindowdump"
        ]

    def parse(self, full_path: str):

        info = super().parse(full_path)

        try:
            with open(full_path, "rb") as image_file:
                with Image.open(image_file) as image:
                    info["mode"] = image.mode
                    info["format_name"] = image.format
                    info["width"] = image.width
                    info["height"] = image.height
        except (OSError, ValueError):
            pass

        return info

class TextFileParser(GenericFileParser):
    is_default = False

    def __init__(self, checksum_calculators: list, content_length: int, root_dir):
        super().__init__(checksum_calculators, root_dir)
        self.content_length = content_length

        self.mime_types = [
            "text/asp", "text/css", "text/ecmascript", "text/html", "text/javascript",
            "text/mcf", "text/pascal", "text/plain", "text/richtext", "text/scriplet",
            "text/sgml", "text/tab-separated-values", "text/uri-list", "text/vnd.abc",
            "text/vnd.fmi.flexstor", "text/vnd.rn-realtext", "text/vnd.wap.wml",
            "text/vnd.wap.wmlscript", "text/webviewhtml", "text/x-asm", "text/x-audiosoft-intra",
            "text/x-c", "text/x-component", "text/x-fortran", "text/x-h", "text/x-java-source",
            "text/x-la-asf", "text/x-m", "text/x-pascal", "text/x-script",
            "text/x-script.csh", "text/x-script.elisp", "text/x-script.guile",
            "text/x-script.ksh", "text/x-script.lisp", "text/x-script.perl",
            "text/x-script.perl-module", "text/x-script.phyton", "text/x-script.rexx",
            "text/x-script.scheme", "text/x-script.sh", "text/x-script.tcl",
            "text/x-script.tcsh", "text/x-script.zsh", "text/x-server-parsed-html",
            "text/x-setext", "text/x-sgml", "text/x-speech", "text/x-uil",
            "text/x-uuencode", "text/x-vcalendar", "text/xml", "text/x-csrc", "text/csv",
            "text/x-c++src", "text/x-chdr", "text/markdown", "text/x-sh", "text/x-java",
            "text/x-python", "text/x-c++hdr", "text/x-tex", "text/x-diff", "text/x-haskell",
            "text/x-perl", "text/x-dsrc", "text/scriptlet", "text/x-scala", "text/calendar",
            "text/x-bibtex", "text/x-tcl", "text/x-c++", "text/x-shellscript", "text/x-msdos-batch",
            "text/x-makefile", "text/rtf", "text/x-objective-c", "text/troff", "text/x-m4",
            "text/x-lisp", "text/x-php", "text/x-gawk", "text/x-awk", "text/x-ruby", "text/x-po",
            "text/x-makefile", "application/javascript", "application/rtf", "application/json",
        ]

    def parse(self, full_path: str):
        info = super().parse(full_path)

        if self.content_length > 0:
            with open(full_path, "rb") as text_file:
                raw_content = text_file.read(self.content_length)

                chardet.detect(raw_content)
                encoding = chardet.detect(raw_content)["encoding"]

                if encoding is not None:
                    info["encoding"] = encoding
                    try:
                        content = raw_content.decode(encoding, "ignore")
                        info["content"] = content
                    except Exception:
                        print("Unknown encoding: " + encoding)

        return info

class FontParser(GenericFileParser):
    is_default = False

    def __init__(self, checksum_calculators: list, root_dir):
        super().__init__(checksum_calculators, root_dir)

        self.mime_types = [
            "application/font-sfnt", "application/font-woff", "application/vdn.ms-fontobject",
            "application/x-font-ttf"
        ]

    def parse(self, full_path: str):

        info = super().parse(full_path)

        with open(full_path, "rb") as f:

            with warnings.catch_warnings():
                warnings.simplefilter("ignore")

                try:
                    font = TTFont(f)

                    if "name" in font:
                        try:
                            for name in font["name"].names:
                                if name.nameID == 4:
                                    info["content"] = name.toUnicode("replace")
                                    break
                        except AssertionError:
                            print("Could not read font name for " + full_path)
                except TTLibError:
                    print("Could not read font for " + full_path)

        return info

class PdfFileParser(GenericFileParser):
    is_default = False

    def __init__(self, checksum_calculators: list, content_length: int, root_dir):
        super().__init__(checksum_calculators, root_dir)

        self.content_length = content_length

        self.mime_types = [
            "application/pdf", "application/x-pdf"
        ]

    def parse(self, full_path: str):
        info = super().parse(full_path)

        if self.content_length > 0:
            with open(full_path, "rb") as f:

                try:
                    parser = PDFParser(f)
                    document = PDFDocument(parser)
                except PDFSyntaxError:
                    print("couldn't parse PDF " + full_path)
                    return info

                info["content"] = ""
                if len(document.info) > 0 and "Title" in document.info[0] and document.info[0]["Title"] != b"":
                    if isinstance(document.info[0]["Title"], bytes):
                        info["content"] += document.info[0]["Title"].decode("utf-8", "replace") + "\n"
                    else:
                        info["content"] += document.info[0]["Title"].resolve().decode("utf-8", "replace") + "\n"

                try:
                    if document.is_extractable:
                        resource_manager = PDFResourceManager()
                        la_params = LAParams()

                        device = PDFPageAggregator(resource_manager, laparams=la_params)
                        interpreter = PDFPageInterpreter(resource_manager, device)

                        for page in PDFPage.create_pages(document):

                            interpreter.process_page(page)
                            layout = device.get_result()

                            for lt_obj in layout:
                                if isinstance(lt_obj, LTTextBox) or isinstance(lt_obj, LTTextLine):

                                    text = lt_obj.get_text()

                                    if len(info["content"]) + len(text) <= self.content_length:
                                        info["content"] += text
                                    else:
                                        info["content"] += text[0:self.content_length - len(info["content"])]
                                        break
                            else:
                                continue
                            break
                    else:
                        print("PDF is not extractable: " + full_path)
                except ValueError:
                    print("Couldn't parse page for " + full_path)

        return info

class EbookParser(GenericFileParser):
    is_default = False

    def __init__(self, checksum_calculators: list, content_length: int, root_dir):
        super().__init__(checksum_calculators, root_dir)

        self.content_length = content_length

        self.mime_types = [
            "application/epub+zip"
        ]

        self.html2text = html2text.HTML2Text()
        self.html2text.ignore_images = True
        self.html2text.ignore_emphasis = True

    def parse(self, full_path: str):
        info = super().parse(full_path)

        book = epub.read_epub(full_path)

        info["content"] = ""

        for text in book.get_items_of_type(ebooklib.ITEM_DOCUMENT):

            text = self.html2text.handle(text.content.decode("utf-8"))

            if len(info["content"]) + len(text) <= self.content_length:
                info["content"] += text
            else:
                info["content"] += text[0:self.content_length - len(info["content"])]
                break

        return info

class DocxParser(GenericFileParser):
    is_default = False

    def __init__(self, checksum_calculators: list, content_length: int, root_dir):
        super().__init__(checksum_calculators, root_dir)

        self.content_length = content_length

        self.mime_types = [
            "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
        ]

    def parse(self, full_path: str):
        info = super().parse(full_path)

        if self.content_length > 0:
            try:
                text = docx2txt.process(full_path)

                if len(text) < self.content_length:
                    info["content"] = text
                else:
                    info["content"] = text[0:self.content_length]
            except:
                print("Couldn't parse Ebook: " + full_path)

        return info

class SpreadSheetParser(GenericFileParser):
    is_default = False

    def __init__(self, checksum_calculators: list, content_length: int, root_dir):
        super().__init__(checksum_calculators, root_dir)

        self.content_length = content_length

        self.mime_types = [
            "application/vnd.ms-excel", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
        ]

    def parse(self, full_path: str):
        info = super().parse(full_path)

        # The MIT License (MIT)
        # Copyright (c) 2014 Dean Malmgren
        # https://github.com/deanmalmgren/textract/blob/master/textract/parsers/xlsx_parser.py

        try:
            workbook = xlrd.open_workbook(full_path)

            sheets_name = workbook.sheet_names()
            info["content"] = ""

            for names in sheets_name:
                worksheet = workbook.sheet_by_name(names)
                num_rows = worksheet.nrows
                num_cells = worksheet.ncols

                for curr_row in range(num_rows):
                    new_output = []
                    for index_col in xrange(num_cells):
                        value = worksheet.cell_value(curr_row, index_col)
                        if value:
                            if isinstance(value, (int, float)):
                                value = six.text_type(value)
                            new_output.append(value)

                    if new_output:
                        text = u' '.join(new_output) + u'\n'
                        if len(info["content"]) + len(text) <= self.content_length:
                            info["content"] += text
                        else:
                            info["content"] += text[0:self.content_length - len(info["content"])]
                            break

            return info

        except xlrd.biffh.XLRDError:
            print("Couldn't parse spreadsheet: " + full_path)


from flask import request, jsonify
from sqlalchemy.exc import PendingRollbackError

from app.exception.InvalidDataError import InvalidDataError
from app.models.categories_models import CategoriesModel
from app.models.eisenhower_model import EisenhowerModel
from app.models.task_model import TaskModel
from app.configs.database import db

def create_task():
    data = request.get_json()
    importance = data["importance"]
    urgency = data["urgency"]
    EisenhowerModel.create_eisenhower()

    data_keys = data.keys()
    keys = TaskModel.keys
    wrong_keys = data_keys - keys

    if len(wrong_keys) > 0:
        return {"error": f"wrong key: {wrong_keys}"}, 400

    try:
        data["name"] = data["name"].title()
        type_eisenhower = TaskModel.eisenhower_type(importance, urgency)

        eisenhower = EisenhowerModel.query.filter_by(type=type_eisenhower).first()
        data["eisenhower_id"] = eisenhower.id

        add_categories = data.pop("categories")

        task = TaskModel(**data)

        for item in add_categories:
            try:
                category = CategoriesModel.query.filter_by(name=item.title()).one()
                task.categories.append(category)

            except:
                new_category = CategoriesModel(name=item.title())
                db.session.add(new_category)
                db.session.commit()

                category = CategoriesModel.query.filter_by(name=item.title()).first()
                task.categories.append(category)

        db.session.add(task)
        db.session.commit()

        return jsonify({
            "id": task.id,
            "name": task.name,
            "description": task.description,
            "duration": task.duration,
            "classification": eisenhower.type,
            "categories": [category.name for category in task.categories]
        }), 201

    except (InvalidDataError) as e:
        return {"msg": e.__dict__["description"]}, 400

    except PendingRollbackError as e:
        name = data["name"]
        return jsonify({"error": f"name: {name}; already registered"}), 409

    except KeyError as e:
        return jsonify({
            "msg":{
                "valid_options": {
                    "importance": [1, 2],
                    "urgency": [1, 2]
                },
                "recieved_options":{
                    "importance": importance,
                    "urgency": urgency
                }
            }
        }), 400

def update_task(id):
    data  = request.get_json()
    data_keys = data.keys()
    keys = TaskModel.keys
    wrong_keys = data_keys - keys

    if len(wrong_keys) > 0:
        return {"error": f"wrong key: {wrong_keys}"}, 400

    try:
        task = TaskModel.query.get(id)

        if task == None:
            return {"error": "task not found!"}, 404

        for key, value in data.items():
            setattr(task, key, value)

        type_eisenhower = TaskModel.eisenhower_type(task.importance, task.urgency)
        eisenhower = EisenhowerModel.query.filter_by(type=type_eisenhower).first()
        data["eisenhower_id"] = eisenhower.id

        db.session.add(task)
        db.session.commit()

        return jsonify({
            "id": task.id,
            "name": task.name,
            "description": task.description,
            "duration": task.duration,
            "classification": eisenhower.type,
            "categories": [category.name for category in task.categories]
        }), 200

    except TypeError as e:
        return jsonify({
            "msg":{
                "valid_options": {
                    "importance": [1, 2],
                    "urgency": [1, 2]
                },
                "recieved_options":{
                    "importance": data["importance"],
                    "urgency": data["urgency"]
                }
            }
        }), 400

    except InvalidDataError as e :
        return {"error": e.args[0]}, 400

def delete_task(id):
    task_to_delete = TaskModel.query.get(id)

    if task_to_delete == None:
            return {"error": "task not found!"}, 404

    db.session.delete(task_to_delete)
    db.session.commit()

    return "", 204


"""
科学与未来 爬虫
"""

import scrapy
import time
import logging
import re
from riskweb.items import ArticleItem
logger = logging.getLogger(__name__)

class MediaAftenpostenSpider(scrapy.Spider):
    name = 'media_sciencesetavenir'
    allowed_domains = ['sciencesetavenir.fr']

    def start_requests(self):
        BASEURL = "https://www.sciencesetavenir.fr"
        start_paths = [
            '/espace/',
            '/sante/',
            '/nutrition/',
            '/nature-environnement/',
            '/animaux/',
            '/high-tech/',
            '/archeo-paleo/'
        ]
        # 发送请求
        for start_path in start_paths:
            yield scrapy.Request(BASEURL+start_path, self.parse_dictionary,meta={'url':start_path}, dont_filter=True)

    def parse_dictionary(self, response):
        BASEURL = "https://www.sciencesetavenir.fr"
        #文章详情链接
        if not re.search( r'page', response.url):
            article_urls = response.xpath('//div[@class="alaune"]/div[@class="txt"]/a/@href | //div[@class="overlay-hover"]/a[1]/@href').extract()
            article_urls.extend(response.xpath('//div[@class="content-main"]//ul[@class="list-img list-img-sqr"]/li/a/@href').extract())
            for article_url in article_urls:
                yield scrapy.Request(article_url, self.parse_article,meta={'topic':response.meta['url'][1:-1]}, dont_filter=False)
            next_url = BASEURL + response.meta['url'] + 'page-' + '2'
            yield scrapy.Request(next_url,self.parse_dictionary,meta={'url':response.meta['url'],'page':3},dont_filter=True)
        else:
            if response.xpath('//div[@class="content-main"]//div[@class="item"]'):
                article_urls = response.xpath('//div[@class="item"]/a/@href').extract()
                for article_url in article_urls:
                    yield scrapy.Request(article_url, self.parse_article,meta={'topic':response.meta['url'][1:-1]}, dont_filter=False)
                next_url = BASEURL + response.meta['url'] + 'page-' + str(response.meta['page'])
                yield scrapy.Request(next_url,self.parse_dictionary,meta={'url':response.meta['url'],'page':response.meta['page']+1},dont_filter=True)

        #scrapy crawl media_sciencesetavenir
    def parse_article(self, response):
        #调取参数
        item = ArticleItem()
        #进入每篇文章获取标题
        item['title'] = response.xpath('//div[@class="article-start"]/h1/text()').extract_first().replace('\xa0','')
        item['media_name'] = '科学与未来'
        item['url'] = response.url
        item['topic'] ='|'.join(response.xpath('//p[contains(@class,"article-tags")]/a/text()').extract()).strip()
        item['author'] = '|'.join(response.xpath('//*[@class="signature"]/text()').extract()).strip()

        # 匹配文章的发布时间
        publish_date = response.xpath('//*[contains(@class,"article-infos")]//a/text()').extract()[-1]
        #10.08.2021 à 11h25

        item['publish_date'] = time.strftime("%Y/%m/%d %H:%M", time.strptime(publish_date, "%d.%m.%Y à %Hh%M")) #08.08.2021 à 16h00

        raw_sentences = response.xpath('//div[contains(@class,"corps")]//p | //div[contains(@class,"corps")]//h2').xpath('string(.)').extract()

        raw_sentences = list(filter(lambda sentence: sentence.strip(), raw_sentences))
        item['content'] = '\n'.join(raw_sentences).strip().replace('\xa0','')
        return item


import imp
from app.helper.shippingdatapagination import ShippingdataPaginate
from app.models.sdct import SdctModel
from app.msmodel.shippingmodel import  Shippingmodel
from app.schemas.shipping import ShippingdataSchema
from app.service.shippingservice import ShippingService
from app.validator.shippingvalidater import Shippingvalidater
from flask import jsonify, request, session
from flask_restplus import Resource, fields, Namespace
import validator
import os
import bcrypt
import json
from app.db import db
import sys, os
from app.models.users import UsersModel
from app.helper.decorator import decorator
from webargs import fields, validate
from webargs.flaskparser import use_args, use_kwargs, parser, abort
from app.helper.errorhandling import Errorhandling

SHIPPINGS_DATA_NOT_FOUND = "No Shipping data found."
SHIPPING_DATA_NOT_FOUND = "Invoice Not found."

shipping_data_ns = Namespace('shipping-data', description='Shipping data related operations')
shipping_datas_ns = Namespace('shipping-datas', description='Shipping datas related operations')

shipping_schema = ShippingdataSchema()
shippings_list_schema = ShippingdataSchema(many=True)
shippings_pagination = ShippingdataPaginate()
shipping_validator = Shippingvalidater()
error_handling = Errorhandling()

shipping_service = ShippingService()

class ShippingsdataList(Resource):
    shippinglist_args = {
        "filter": fields.Str(required=False),
        "order_by": fields.Str(missing="SDCT_ID",required=False),
        "order_direction": fields.Str(missing="Asc", validate=validate.OneOf(["Asc", "Desc","asc","desc"])),
        "limit": fields.Int(missing=25, required=False),
        "ship_id": fields.Int(required=False),
        "page": fields.Int(missing=0, required=False),
        "active": fields.Str(
            missing="all", validate=validate.OneOf(["active", "inactive" , "all"])
        )
    }
    @use_kwargs(shippinglist_args, location="query")
    @shipping_datas_ns.doc('Get all shipping data')
    def get(self, **kwargs):
        filter_data = {}
        if "order_direction" in kwargs:
            order_direction = kwargs['order_direction']

        if "order_by" in kwargs:
            order_by = kwargs['order_by']
        if "ship_id" in kwargs:
            ship_id = kwargs['ship_id']
            filter_data["ship_id"] = ship_id
        if  kwargs['active'] != "all":
            active = kwargs['active']
            filter_data["active"] = active

        if "filter" in kwargs:
            filter_data["filter"] = kwargs['filter']

        shippings = SdctModel.find_all_by_filter(filter_data, order_by, order_direction)
        if len(shippings) == 0:
            return {'message':  SHIPPINGS_DATA_NOT_FOUND}, 404
        shipping_list = shippings_list_schema.dump(shippings)

        shippings = shippings_pagination.paginator(shipping_list, kwargs['page'], kwargs['limit'],  len(shipping_list))
        return shippings, 200

class Shippingdata(Resource):
    @shipping_data_ns.doc('Get Shipping Detail')
    def get(self, id):
        shipping = SdctModel.find_by_id(id)

        if shipping:
            shipping = shipping_schema.dump(shipping)
            return shipping
        return {'message':  SHIPPINGS_DATA_NOT_FOUND}, 404

    @shipping_data_ns.doc('Create a Shipping')
    def post(self):
        data = request.get_json(force=True)

        rule = shipping_validator.rule()

        result, req_obj, errors = validator.validate(data, rule, return_info=True)

        if result == False:
            error = error_handling.error(errors, data)
            return error, 400

        shipping_data = Shippingmodel(**data)
        add_shipping = shipping_service.create(shipping_data)

        if add_shipping:
            return shipping_schema.dump(add_shipping)

        return {'message':  'Unable to save Shipping data'}, 400


import pygame,sys
from scripts.character import Character
from scripts.animations import Animations
from scripts.sword import Sword
from scripts.playerInput import PlayerInput
class Player(Character):
    def __init__(self,name,x, y,hp, width, height,collisionRects,hpBarImg=None):#init the player
        super().__init__(x, y, width, height,collisionRects)#init the character class
        self.name = name#name of the player
        self.weapons = {'sword' : Sword(x,y)}#the weapons the player has
        self.takeInputs = True#does the player take inputs
        self.attacking = False#is the player attacking
        self.dash = False#is the player dashing
        self.slide = False#is the player sliding
        self.dead = False#is the player dead
        self.input = PlayerInput(self)#the player's input
        self.slideVel = None#the velocity of the player when sliding
        self.slideAcc = None#the acceleration of the player when sliding
        self.imunityFrames = 0#the amount of frames the player is immune to damage
        self.dashAcceleration = 0.5#the acceleration of the player when dashing
        self.attackCombo = 'firstAttack'#the attack combo the player is on
        self.hpBar = self.getHpBar(hp,hpBarImg)#the hp bar of the player
        self.animations = self.getAnimations('assets/playerAnimations',run=[4,4,4,4,4],#the animations of the player
                                             idle=[1],firstAttack=[4,4,4,4,4,4],secondAttack=[5,5,4,4],
                                             thirdAttack=[5,10,4],dash=[4,5,5,5,5,5])

    @property
    def isInIngameMenu(self):#is the player in the in game menu
        return self.input.runInGameMenu

    def drawPlayerHpBar(self,gameSurface):#draw the player's hp bar
        pygame.draw.rect(gameSurface,(172,50,50),(9,9+170,self.hpBarWidth-10,self.hpBar['height']-10))#draw the red hp bar
        gameSurface.blit(self.hpBar['img'],(4,4+170))#draw the hp bar image

    def attack(self):#attack function
        if self.attackCombo =='firstAttack':#if the attack combo is the first attack
            self.triggerAttack1()#call the triggerAttack1 function
        elif self.attackCombo == 'secondAttack':#if the attack combo is the second attack
            self.triggerAttack2()#call the triggerAttack2 function
        elif self.attackCombo == 'thirdAttack':#if the attack combo is the third attack
            self.triggerAttack3()#call the triggerAttack3 function

    def triggerAttack1(self):#trigger the first attack
        self.animations.changeState('firstAttack')#change the animation state to firstAttack
        currentImg = self.animations.getCurrentImg#get the current image of the animation
        if currentImg == 'firstAttack2':#if the current image is the second image of the first attack
            self.weapons["sword"].arc = self.weapons["sword"].newBezeirArc(self.x,self.y,self.flip,'down')#create a new bezeir arc
            self.startSlide()#start the slide
        elif currentImg == 'firstAttack5':#if the current image is the fifth image of the first attack
            self.attacking = False#stop attacking
            self.attackCombo = 'secondAttack'#change the attack combo to the second attack

    def triggerAttack2(self):#trigger the second attack
        self.animations.changeState('secondAttack')#change the animation state to secondAttack
        currentImg = self.animations.getCurrentImg#get the current image of the animation
        if currentImg == 'secondAttack2':#if the current image is the second image of the second attack
            self.weapons["sword"].arc = self.weapons["sword"].newBezeirArc(self.x,self.y,self.flip,'up')#create a new bezeir arc
            self.startSlide()#start the slide
        elif currentImg == 'secondAttack3':#if the current image is the third image of the second attack
            self.attacking = False#stop attacking
            self.attackCombo = 'thirdAttack'#change the attack combo to the third attack
    def triggerAttack3(self):#trigger the third attack
        self.animations.changeState('thirdAttack')#change the animation state to thirdAttack
        currentImg = self.animations.getCurrentImg#get the current image of the animation
        if currentImg == 'thirdAttack0':#if the current image is the first image of the third attack
            self.startSlide()#start the slide
            self.weapons["sword"].arc = self.weapons["sword"].newBezeirArc(self.x,self.y,self.flip,'down')#create a new bezeir arc
        elif currentImg == 'thirdAttack2':#if the current image is the third image of the third attack
            self.attacking = False#stop attacking
            self.attackCombo = 'firstAttack'#change the attack combo to the first attack
    def triggerDash(self):#trigger the dash
        self.attacking = False#stop attacking
        if self.flip: #if the player is flipped
            self.velocity[0] = -(6 / self.dashAcceleration)#set the velocity to the dash acceleration
            self.dashAcceleration += 0.5#increase the dash acceleration
        else: #if the player is not flipped
            self.velocity[0] = (6 / self.dashAcceleration)#set the velocity to the dash acceleration
            self.dashAcceleration += 0.5#increase the dash acceleration
        self.animations.changeState('dash')#change the animation state to dash
        currentImg = self.animations.getCurrentImg#get the current image of the animation
        if currentImg == 'dash5':#if the current image is the fifth image of the dash
            self.dash = False#stop dashing
            self.dashAcceleration = 0.5#reset the dash acceleration

    def changeAnimationState(self,movement):#change the animation state
        if movement[0] == 0:self.animations.changeState('idle')#if the player is not moving change the animation state to idle
        if movement[0] > 0:self.animations.changeState('run')#if the player is moving right change the animation state to run
        if movement[0] < 0:self.animations.changeState('run')#if the player is moving left change the animation state to run

    def kill(self):#kill the player
        self.hpBar['hp'] = 100#reset the hp
        self.dead = True#set the player to dead

    def startSlide(self):#start the slide
        self.slideVel = 3#set the slide velocity
        self.slideAcc = -1#set the slide acceleration
        self.slide = True#set the slide to true

    def slideUpdate(self):#update the slide
        if self.slide:#if the player is sliding
            self.slideVel += self.slideAcc#increase the slide velocity
            if self.flip: self.x -=self.slideVel#move the player left
            else: self.x += self.slideVel#move the player right
            if self.slideVel <= 0:#if the slide velocity is less than or equal to 0
                self.slide = False#stop sliding
                self.slideVel = None#reset the slide velocity
                self.slideAcc = None#reset the slide acceleration

    def update(self,gameSurface,scroll, enemies):#update the player
        self.updateCharacter()#update the character
        self.input.update()#update the input
        if not self.attacking:#if the player is not attacking
            movement = self.move(self.collisionRects)#move the player
        if self.dash: self.triggerDash()#if the player is dashing call the triggerDash function
        elif self.attacking: #if the player is attacking
            self.attack()#call the attack function
        else: self.changeAnimationState(movement)#change the animation state

        self.slideUpdate()#update the slide
        add = 0#set the add variable to 0
        if self.animations.getCurrentImg[:-1] == 'thirdAttack':#if the current image is the third attack
            if self.flip: add = -13#set the add variable to -13
            else: add = 13#set the add variable to 13
        self.weapons['sword'].update(self.x+add,self.y,gameSurface,scroll,enemies)#update the sword add to the x position of the player so that it's in the correct postion depending on the animation

        self.draw(gameSurface,scroll,self.animations.getImg())#draw the player
        self.drawPlayerHpBar(gameSurface)#draw the player hp bar


from collections import deque
from enum import Enum, auto

from random import choice, shuffle

# noinspection PyArgumentList
class OP(Enum):
    JUMP = auto()
    DISPLAY = auto()
    PUSH = auto()
    POP = auto()
    LABEL = auto()
    JUMP_IF_TRUE = auto()
    JUMP_IF_FALSE = auto()
    PUSH_RANDOM = auto()
    SHUFFLE = auto()
    BRANCH_IF_TRUE = auto()
    EXIT = auto()
    PUSH_RANDOM_BOOL = auto()
    PRINT_JOIN = auto()
    BRANCH_IF_FALSE = auto()
    SHOW_STACK = auto()
    STORE = auto()
    LOAD = auto()
    DUPLICATE = auto()
    PRINT = auto()
    CLEAR = auto()

def safe_get(items, index, default):
    return items[index] if 0 <= index < len(items) else default

def run(codes):
    stack = deque()
    labels = {}
    variables = {}

    def pop():
        return stack.pop()

    def push(value):
        stack.append(value)

    for i, (opcode, *args) in enumerate(codes):
        if opcode is OP.LABEL:
            labels[args[0]] = i

    codes_length = len(codes)
    ptr = 0
    while 0 <= ptr < codes_length:
        code, *args = codes[ptr]
        if code is OP.JUMP:
            ptr = labels[args[0]]
            continue
        elif code is OP.PUSH:
            push(args[0])
        elif code is OP.POP:
            pop()
        elif code is OP.DISPLAY:
            print(pop(), end='\n' if safe_get(args, 0, True) else '')
        elif code is OP.JUMP_IF_TRUE:
            if pop():
                ptr = labels[args[0]]
        elif code is OP.JUMP_IF_FALSE:
            if not pop():
                ptr = labels[args[0]]
        elif code is OP.PUSH_RANDOM:
            push(choice(args))
        elif code is OP.SHUFFLE:
            shuffle(stack)
        elif code is OP.BRANCH_IF_TRUE:
            ptr = labels[args[0] if pop() else args[1]]
        elif code is OP.BRANCH_IF_FALSE:
            ptr = labels[args[0] if not pop() else args[1]]
        elif code is OP.EXIT:
            quit(0)
        elif code is OP.PUSH_RANDOM_BOOL:
            push(choice((True, False)))
        elif code is OP.PRINT_JOIN:
            print(' '.join(args))
        elif code is OP.SHOW_STACK:
            print(list(stack))
        elif code is OP.STORE:
            variables[args[0]] = pop()
        elif code is OP.LOAD:
            push(variables[args[0]])
        elif code is OP.DUPLICATE:
            value = pop()
            push(value)
            push(value)
        elif code is OP.PRINT:
            print(args[0], end='\n' if safe_get(args, 1, True) else '')
        elif code is OP.CLEAR:
            del variables[args[0]]
        ptr += 1


import os
from tqdm import tqdm
import logging
import jsonlines

from retrieving.retrieval_module import get_retrieval_module
from utils import Arguments, set_seed, print_args
from utils.time_log import TimeMeasure
from evaluation.metrics import Metrics

logger = logging.getLogger(__name__)

def main():
    parser = Arguments()
    parser.add_model_parameters()
    parser.add_retrieval_module_parameters()

    parser.add_argument("--checkpoint_path", type=str, required=True)
    parser.add_argument("--query_file", type=str, required=True,
                        help="query file is jsonl file")
    parser.add_argument("--document_name", type=str, required=True)
    parser.add_argument("--faiss_depth", type=int, default=1)
    parser.add_argument("--batch_size", type=int, default=32)
    args = parser.parse()

    print_args(args, logger)
    set_seed(args.seed)

    logging.basicConfig(
        format="%(asctime)s - %(levelname)s - %(name)s -   %(message)s",
        datefmt="%m/%d/%Y %H:%M:%S",
        level=logging.INFO,
    )

    # Load retrieving Model and files
    logger.info("[Load retrieval_module]===========")
    retrieval = get_retrieval_module(args)

    document_path = os.path.join(args.index_dir, f"{args.document_name}.bin")
    if os.path.exists(document_path):
        retrieval.load_document(document_path)
    else:
        logger.info(f"Wrong document name")
        exit(0)

    qas = []
    with jsonlines.open(args.query_file, "r") as f:
        for line in f:
            qas.append(line)

    # Evaluate
    time_measure = TimeMeasure(logger)
    metric = Metrics(mrr_depths={10, 50, 100},
                     success_depths={10, 50, 100},
                     recall_depths={10, 50, 100})

    with time_measure as tm:
        tm.set_prefix("[Do Evaluate]===========: ")

        q_id = 0
        for offset in tqdm(range(0, len(qas), args.batch_size)):
            endpos = min(offset + args.batch_size, len(qas))
            batch = qas[offset: endpos]

            queries = [b['query'] for b in batch]
            _, pids = retrieval.retrieval(queries, args.document_name,
                                          faiss_depth=args.faiss_depth, verbose=args.retrieval_verbose)

            for i in range(len(batch)):
                metric.add(q_id, pids[i], batch[i]['answers'])
                q_id += 1

        metric.print_metrics()

if __name__ == "__main__":
    main()


###### NEURAL NETWORK BASED VOTING SYSTEM ########
import csv

import numpy as np
from sklearn.externals import joblib
from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split
from sklearn.neural_network import MLPClassifier

def vote(logreg, gaussian, linearsvc, Y1):
    X = np.array(list(zip(logreg, gaussian, linearsvc)))
    Y = np.array(Y1)

    x_train, x_test, y_train, y_test = train_test_split(X, Y, test_size=0.25)
    clf = MLPClassifier(solver='adam', alpha=1e-3, hidden_layer_sizes=(32, 16), batch_size=16, learning_rate='adaptive', learning_rate_init=0.001, verbose=True)
    clf.fit(x_train, y_train)
    print('ANN training completed')
    y_real, y_pred = y_test, clf.predict(x_test)

    joblib.dump(clf, 'models/know_ann_voter.pkl')

    print('Accuracy: {:.2f}%'.format(accuracy_score(y_real, y_pred) * 100))
    '''
    print('MaxEnt Accuracy: {:.2f}%'.format(accuracy_score(y_real, x_test.T[0]) * 100))
    print('BiRNN Accuracy: {:.2f}%'.format(accuracy_score(y_real, x_test.T[1]) * 100))
    print('SVM-GloVe Accuracy: {:.2f}%'.format(accuracy_score(y_real, x_test.T[2]) * 100))
    '''
know_labels = {'Factual':0, 'Conceptual':1, 'Procedural':2, 'Metacognitive':3}
if __name__ == '__main__':
    VOTE_SUB = 'ADA'
    questions = []
    logreg = []
    gaussian = []
    linearsvc = []
    Y1 = []
    if VOTE_SUB == 'ADA':
        with open('datasets/ADA_Exercise_Questions_Results.csv', encoding="utf-8") as csvfile:
            csvreader = csv.reader(csvfile.read().splitlines()[1:])
            for row in csvreader:
                questions.append(row[0])
                Y1.append(know_labels.get(row[1]))
                logreg.append(know_labels.get(row[2]))
                gaussian.append(know_labels.get(row[3]))
                linearsvc.append(know_labels.get(row[4]))

        vote(logreg, gaussian, linearsvc, Y1)

    if VOTE_SUB == 'Combined':
        with open('datasets/COMBINED_Exercise_Questions_Results1.csv', encoding="utf-8") as csvfile:
            csvreader = csv.reader(csvfile.read().splitlines()[1:])
            for row in csvreader:
                questions.append(row[0])
                Y1.append(know_labels.get(row[1]))
                logreg.append(know_labels.get(row[2]))
                gaussian.append(know_labels.get(row[3]))
                linearsvc.append(know_labels.get(row[4]))

        vote(logreg, gaussian, linearsvc, Y1)


#!/usr/bin/python3
#Given a folder with ecoding, instances and list of instances run IDLV for each instances and
#    run all encodings and save the running time

import sys
import os
import numpy as np
import datetime

import annotator
import runner

PID = os.getpid()
WARNING_PERC = 30

def extractStats(dir, encoding_means):
    stats = []
    warnings = []
    if 0 not in encoding_means:
        print("ERROR default encoding not in %s" % dir)
        return stats

    problem = runner.getLast(dir)
    base_time = encoding_means[0]
    for e in encoding_means:
        p = ((base_time - encoding_means[e]) / base_time) * 100
        if p > WARNING_PERC:
            warnings.append("%s\t%s\t%s\t%s" % (problem, e, str(encoding_means[e]), str(p)))
        stats.append("%s\t%s\t%s\t%s" % (problem, e, str(encoding_means[e]), str(p)))

    return (stats, warnings)

def main(directory):
    print("Start extract stats")
    directoies = []
    for dir in os.listdir(directory):
        directoies.append(directory+dir)

    dir_out = {}
    for dir in directoies:
        dir_out[dir] =  dir +"/" + annotator.checkDir(dir)

    fileOut = open("result_"+str(datetime.date.today())+"_"+str(PID),"w")
    fileSum = open("result_summary"+str(datetime.date.today())+"_"+str(PID),"w")
    fileOut.write("Problem\tencoding\ttime(s)\t%\n")
    for dir in dir_out:
        do = dir_out[dir]
        file = open(do+"/"+runner.RESULTFILE, "r")
        encoding_times = {}
        for line in file:
            num = int(line.split("\t")[0].split(".")[-1])
            if num not in encoding_times:
                encoding_times[num] = []
            encoding_times[num].append(float(line.split("\t")[2]))
        encoding_means = {}
        for k in encoding_times:
            encoding_means[k] = np.mean(encoding_times[k])

        s, w = extractStats(dir, encoding_means)
        for l in s:
            fileOut.write(l+"\n")
        file.close()
        for l in w:
            fileSum.write(l+"\n")
    fileOut.close()
    fileSum.close()

if __name__ == "__main__":
    folder = sys.argv[1]
    if folder[-1] != '/':
        folder += '/'
    main(folder)