# -*- coding: utf-8 -*-
#
# author: Amanul Haque
#
# File Description: This code contained methods fot supervised classification algorithms

from word_to_vec_vectorizer import vectorization
from sklearn import svm
from sklearn.svm import SVC
from sklearn.svm import LinearSVC
from sklearn.linear_model import LogisticRegression
from sklearn.naive_bayes import MultinomialNB
from sklearn import metrics
from Semi_EM_NB import Semi_EM_MultinomialNB
from scipy import sparse
from sklearn.semi_supervised import LabelPropagation
from compare_labels_probabilities import compare_labels_probabilities
from sklearn.tree import DecisionTreeClassifier
import numpy as np
from sklearn.ensemble import RandomForestClassifier
import sys
from scipy import sparse
from sklearn.semi_supervised import LabelSpreading
from transductive_SVM import transductive_SVM

class classification:

    def __init__(self):
        self.a = 0

    def classifier_1(self, X_train, y_train, X_test, labelled_set, unlabelled_set):
        clf = svm.SVC(kernel = 'linear', C=1.0, probability = True)
        clf.fit(X_train, y_train)
        predicted_labels = clf.predict(X_test)
        predicted_probabilities = clf.predict_proba(X_test)
        return predicted_labels, predicted_probabilities

    def linear_svc(self, X_train, y_train, X_test):
        clf = LinearSVC(random_state=0, tol=1e-5)
        clf.fit(X_train, y_train)
        predicted_labels = clf.predict(X_test)
        prediction_confidence = clf.decision_function(X_test)
        return predicted_labels, prediction_confidence, clf

    def radial_svc(self, X_train, y_train, X_test):
        clf = SVC(gamma='auto', kernel='rbf')
        clf.fit(X_train, y_train)
        predicted_labels = clf.predict(X_test)
        prediction_confidence = clf.decision_function(X_test)
        return predicted_labels, prediction_confidence, clf

    def logistic_regression(self, X_train, y_train, X_test):
        clf = LogisticRegression(random_state=0, solver='lbfgs', multi_class='multinomial').fit(X_train, y_train)
        predicted_labels = clf.predict(X_test)
        prediction_confidence = clf.decision_function(X_test)
        final_labels = clf.predict(X_test)
        label_prob = clf.predict_proba(X_test)
        print(compare_labels_probabilities().compare(label_prob, final_labels))
        return predicted_labels, prediction_confidence, clf

    #def calibration_isotonic(self, X_train, y_train, X_test, clf)

    def expectation_maximization(self,  X_train, y_train, X_test):

        nb_clf = MultinomialNB(alpha=1e-2)
        #print("XX ", X_train.shape, "\tYY ", y_train.shape)
        nb_clf.fit(X_train, y_train)
        #print("XXXXXX ", X_test.shape)
        #print(X_test)

        # Train Naive Bayes classifier (imported)
        # using both labeled and unlabeled data set
        em_nb_clf = Semi_EM_MultinomialNB(alpha=1e-2) # semi supervised EM based Naive Bayes classifier
        #X_test = np.asmatrix(X_test)
        #X_test = sparse.csr_matrix(X_test)
        #print("TYPE : ", type(X_test))
        em_nb_clf.fit(X_train, y_train, X_test)
        # em_nb_clf.fit_with_clustering(X_l, y_l, X_u)
        # em_nb_clf.partial_fit(X_l, y_l, X_u)

        '''
        # Evaluate original NB classifier using test data set
        pred = nb_clf.predict(X_test)
        print("Normal Naive Bayes")
        print(pred)
        print(metrics.classification_report(test_Xy.target, pred, target_names=['1','2']))
        # pprint(metrics.confusion_matrix(test_Xy.target, pred))
        print(metrics.accuracy_score(test_Xy.target, pred))
        '''

        # Evaluate semi-supervised EM NB classifier using test data set
        pred = em_nb_clf.predict(X_test)
        print("EM Semi supervised")
        #print(pred)
        #unique, counts = np.unique(pred, return_counts=True)
        #print(dict(zip(unique, counts)))
        return pred, em_nb_clf
        #sys.exit()
        #print(metrics.classification_report(test_Xy.target, pred, target_names=['1','2']))
        # pprint(metrics.confusion_matrix(test_Xy.target, pred))
        #print(metrics.accuracy_score(test_Xy.target, pred))

        # find the most informative features
        #import numpy as np
        '''
        def show_topK(classifier, vectorizer, categories, K=10):
            feature_names = np.asarray(vectorizer.get_feature_names())
            for i, category in enumerate(categories):
                topK = np.argsort(classifier.coef_[i])[-K:]
                print("%s: %s" % (category, " ".join(feature_names[topK])))

        #show_topK(nb_clf, vectorizer, ['1','2'], K=10) # keywords for each class by original NB classifier

        #show_topK(em_nb_clf, vectorizer, ['1','2'], K=10) # keywords for each class by semisupervised EM NB classifier
        '''
        #print(nb_clf.class_log_prior_, em_nb_clf.clf.class_log_prior_)

    def label_propagation(self, X_train, y, X_test):

        clf = LabelPropagation()
        print("X_train Shape :", X_train.shape, type(X_train))
        print("X_test shape : ", X_test.shape, type(X_test))
        print("y shape : ", y.shape )

        X = np.concatenate((X_train.todense(), X_test.todense()),axis=0)
        print("X shape now ", X.shape)
        print("Y shape now ", y.shape)
        clf.fit(X, y)
        final_labels = clf.predict(X_test)
        label_prob = clf.predict_proba(X_test)
        print(compare_labels_probabilities().compare(label_prob, final_labels))
        return final_labels, clf

    def decision_tree(self, X_train, y_train, X_test):
        clf = DecisionTreeClassifier(random_state=0)
        clf.fit(X_train, y_train)
        final_labels = clf.predict(X_test)
        return final_labels, clf

    def random_forest(self, X_train, y_train, X_test):
        clf = RandomForestClassifier()
        clf.fit(X_train, y_train)
        final_labels = clf.predict(X_test)
        return final_labels, clf

    def label_spreading(self, X_train, y, X_test):
        clf = LabelSpreading()
        X = np.concatenate((X_train.todense(), X_test.todense()),axis=0)
        print("X shape now ", X.shape)
        print("Y shape now ", y.shape)
        clf.fit(X, y)
        final_labels = clf.predict(X_test)
        label_prob = clf.predict_proba(X_test)
        print(compare_labels_probabilities().compare(label_prob, final_labels))
        return final_labels, clf

    def transductive_svm(self, X_train, y, X_test):
        clf = transductive_SVM()
        X = np.concatenate((X_train.todense(), X_test.todense()),axis=0)
        print("X shape now ", X.shape)
        print("Y shape now ", y.shape)
        y = y.astype(int)
        print(y)
        clf.fit(X, y)
        final_labels = clf.predict(X_test)
        #label_prob = clf.predict_proba(X_test)
        #print(compare_labels_probabilities().compare(label_prob, final_labels))
        return final_labels, clf

    def one_class_svm(self, X_train, y_train, X_test):
        clf = svm.OneClassSVM(nu = 0.5, kernel = 'rbf', gamma = 0.1)
        clf.fit(X_train, y_train)
        y_pred = clf.predict(X_test)
        unique, counts = np.unique(y_pred, return_counts=True)
        print(dict(zip(unique, counts)))
        return y_pred, clf

# =============================================================================
# cl = classification()
# X_train, y_train, X_test, labelled_set, unlabelled_set = cl.get_vectorized_data()
# predicted_labels, prediction_confidence = cl.classifier_2(X_train, y_train, X_test, labelled_set, unlabelled_set)
# print(predicted_labels, predicted_labels.shape)
# print(prediction_confidence, prediction_confidence.shape)
# =============================================================================


import time
from cProfile import Profile
from timeit import default_timer
from typing import Dict, Optional

from py_grpc_profile.adapter.interfaces import Adapter

class CProfileAdapter(Adapter):
    DEFAULT_FILENAME_FORMAT = (
        "grpc-server-{service}-{method}-{elapsed:.0f}ms-{time:.0f}.prof"
    )

    def __init__(
        self,
        filename_format: Optional[str] = None,
    ):
        self.filename_format = (
            self.DEFAULT_FILENAME_FORMAT if filename_format is None else filename_format
        )

    def run(self, behavior, request_or_iterator, servicer_context, metadata: Dict):
        profile = Profile()
        start = default_timer()
        profile.enable()

        try:
            response_or_iterator = behavior(
                request_or_iterator,
                servicer_context,
            )
        finally:
            profile.disable()
            elapsed = default_timer() - start
            filename = self.filename_format.format(
                service=metadata["grpc_service_name"],
                method=metadata["grpc_method_name"],
                elapsed=elapsed * 1000.0,
                time=time.time(),
            )
            profile.dump_stats(filename)
        return response_or_iterator


"""updating bio

Revision ID: 1cf6901ea903
Revises: 1837077807ac
Create Date: 2021-01-03 16:20:41.351378

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '1cf6901ea903'
down_revision = '1837077807ac'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    with op.batch_alter_table('bio', schema=None) as batch_op:
        batch_op.add_column(sa.Column('text', sa.String(length=3000), nullable=True))
        batch_op.add_column(sa.Column('timestamp', sa.DateTime(), nullable=True))
        batch_op.create_index(batch_op.f('ix_bio_timestamp'), ['timestamp'], unique=False)
        batch_op.drop_column('body')

    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    with op.batch_alter_table('bio', schema=None) as batch_op:
        batch_op.add_column(sa.Column('body', sa.VARCHAR(length=140), nullable=True))
        batch_op.drop_index(batch_op.f('ix_bio_timestamp'))
        batch_op.drop_column('timestamp')
        batch_op.drop_column('text')

    # ### end Alembic commands ###


from django import forms

class LAForm01(forms.Form):

    def __init__(self, *args, **kwargs):
        if 'qal' in kwargs:
            qal = kwargs.pop('qal')
            super(LAForm01, self).__init__(*args, **kwargs)
            qanum = 0
            for qa in qal:
                #qal is the question answer list which alternates question, answer so
                # %2 is used first (if) to get question then (else) to get answer
                qnum = qanum // 2
                if qanum % 2 == 0:
                    self.fields['answer-' + str(qnum)] = forms.CharField(label=qa, max_length=30, widget=forms.NumberInput)
                    self.fields['hiddenq-' + str(qnum)] = forms.CharField(initial=qa, widget=forms.HiddenInput)
                else:
                    self.fields['hiddena-' + str(qnum)] = forms.CharField(initial=qa, widget=forms.HiddenInput)
                qanum += 1
        else:
            super(LAForm01, self).__init__(*args, **kwargs)

    def q_marks(self, n):
        qm = []
        for i in range(n):
            if self.cleaned_data['answer-' + str(i)] == self.cleaned_data['hiddena-' + str(i)]:
                qm.append("-CORRECT-")
            else:
                qm.append("--WRONG--")
        return qm

"""
solution for multiple/list of fields
 from https://jacobian.org/writing/dynamic-form-generation/
 also see https://stackoverflow.com/questions/17159567/how-to-create-a-list-of-fields-in-django-forms

forms documentation
 working with - https://docs.djangoproject.com/en/1.10/topics/forms/
 reference - https://docs.djangoproject.com/en/1.10/ref/forms/
 field widgets - https://docs.djangoproject.com/en/1.10/ref/forms/widgets/

    def __init__(self, *args, **kwargs):
        extra = kwargs.pop('extra')
        super(UserCreationForm, self).__init__(*args, **kwargs)

        for i, question in enumerate(extra):
            self.fields['custom_%s' % i] = forms.CharField(label=question)
"""


import logging
from telegram.ext import Updater, CommandHandler
from ipgetter2 import IPGetter

#logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')

def ip(update, context):
    getter = IPGetter()
    ips=getter.get()
    update.message.reply_text('Your ips:\n v4:{0}\n v6:{1}'.format(ips.v4,ips.v6))

# main

tokenFile = open("token.txt", "r")
token=tokenFile.readline()

updater = Updater(token, use_context=True)

updater.dispatcher.add_handler(CommandHandler('ip', ip))

updater.start_polling()
updater.idle()


from pydantic import BaseModel, Field, constr, confloat, create_model
from typing import Optional

validate_str = constr(
    max_length=50,
    min_length=3,
    strict=True
)

validate_float = confloat(
    gt=0.0,
    strict=True
)

class BaseProducto(BaseModel):
    nombre: validate_str = Field(...)
    marca: validate_str = Field(...)
    precio: validate_float = Field(...)

    @classmethod
    def as_optional(cls):
        annonations = cls.__fields__
        fields = {attribute: (Optional[data_type.type_], None) for attribute, data_type in annonations.items()}
        OptionalModel = create_model(f"Optional{cls.__name__}", **fields)
        return OptionalModel

class Producto(BaseProducto):
    id: int = Field(...)


import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
import matplotlib.pyplot as plt
import math

np.random.seed(3)

# Constants
learning_rate = 0.0001
num_iters = 4000
bias = False

# Calculate the sigmoid / logistic function
def sigmoid(z):
    return 1 / (1 + np.exp(-z))

# 0-1 accuracy model
def accuracy(y_true, y_pred):
    accuracy = np.sum(y_true == y_pred) / len(y_true)
    return accuracy

# Dataset generation
#   p: number of features
#   m: norm of the parameter vector
#   theta: parameter vector for generation
#   n: number of samples in training set
#   t: number of samples in testing set
#   Generate an n x p array of floats from 0 to 1, and dot with theta to find z
#   Probabilities result from applying sigmoid to the z array
p = 100
m = 30
theta = np.random.random(size=(p, 1)) * 4 - 2
theta = theta * m / np.linalg.norm(theta)

print(f'Norm of parameter vector: {np.linalg.norm(theta)}')
n = 25000
t = 500
X = np.random.rand(n+t, p)
z = np.dot(X, theta)
prob = sigmoid(z)

# Generate labels by sampling from Bernoulli(prob)
y = np.random.binomial(1, prob.flatten())

# Generate labels deterministically
## y = np.where(prob.flatten() >= 0.5, 1, 0)

# Split data into training and testing data
xtrain, xtest, ytrain, ytest = train_test_split(X, y, test_size=t/(n+t), random_state=0)

# Train the model
iter_list = []
rms_list = []

# plot as n increases the RMSE
for iter in range(100, 25000, 100):
    regressor = LogisticRegression()
    regressor.fit(xtrain[:iter], ytrain[:iter])
    print(f'Trained weights with n={iter} has norm {np.linalg.norm(regressor.coef_)} with accuracy {accuracy(ytest, regressor.predict(xtest))}')

    ## Formula 1
    MSE = np.linalg.norm(np.subtract(theta,regressor.coef_.T)) / np.linalg.norm(theta)

    ## Formula 2
    # MSE = np.linalg.norm(np.subtract(theta / np.linalg.norm(theta),regressor.coef_.T / np.linalg.norm(regressor.coef_.T)))

    RMSE = MSE / math.sqrt(p)
    print(f'RMSE: {RMSE}')

    iter_list.append(iter)
    rms_list.append(RMSE)

plt.plot(iter_list, rms_list)
plt.xlabel("Number of iteration")
plt.ylabel("RMSE")
plt.show()


import pygame
import sys
from pygame.locals import *
import csv
import math
import asyncio

pygame.init()
screen = pygame.display.set_mode((1000,1000))
myfont = pygame.font.Font(None, 60)
screen.fill((255,255,255))

inp = ""

def loadData(tag) :

    global screen

    screen.fill((255, 255, 255))

    black = (0, 0, 0)
    white = (255, 255, 255)
    red = (255, 0, 0)
    blue = (0, 0, 255)

    color = black

    file = open("tagdataOld.txt", "r")
    reader = csv.reader(file, delimiter="|")

    x = 0
    y = 0
    counter = 0
    xpos = 0
    ypos = 0

    for i, row in enumerate(reader):

        if (i < 750) :

            if (row[1] == tag) :

                print (i)

                x = float(row[2])
                y = float(row[3])
                x = math.floor(x)
                y = math.floor(y)

                xpos = xpos + x
                ypos = ypos + y

                counter = counter+1

                if (counter == 5) :
                    xpos = (xpos/50)*2
                    ypos = (ypos/50)*2
                    counter = 0
                    print (row[0],str(xpos),str(ypos))
                    pygame.draw.circle(screen, color, (math.floor(xpos), math.floor(ypos)), 2, 2)
                    pygame.display.update()

            '''
            x = float(row[2])/10
            y = float(row[3]+2000)/10
            x = math.floor(x)
            y = math.floor(y)

            xpos = xpos+x
            ypos = ypos+y

            if (row[1] == '27211') :
                color = blue
            if (row[1] == '27245') :
                color = red
            if (row[1] == '27205') :
                color = black
            '''
    '''
    xpos = xpos/10
    ypos = ypos/10

    print (xpos,ypos)
    pygame.draw.circle(screen, color, (math.floor(xpos), math.floor(ypos)), 2, 2)
    pygame.display.update()
    '''

async def detectKeys() :

    line = 0

    while True:
        for event in pygame.event.get():
            if event.type == QUIT:
                sys.exit()
            if event.type == pygame.KEYDOWN:
                    if event.key == pygame.K_1:
                        print ('loading 27211')
                        loadData('27211')
                    if event.key == pygame.K_2:
                        print ('loading 27205')
                        loadData('27205')
                    if event.key == pygame.K_3:
                        print ('loading 27241')
                        loadData('27241')
detectKeys()

async def main():
    detectKeysTask = asyncio.create_task(detectKeys())
    await detectKeysTask

asyncio.run(main())


from pathlib import Path
import sys
import numpy as np
from sklearn.model_selection import train_test_split
import torch
from torch import nn
from torch.utils.data import DataLoader
import pytorch_lightning as pl
from pytorch_lightning.callbacks import ModelCheckpoint, EarlyStopping
import torch.multiprocessing

class Model(torch.nn.Module):
    def __init__(self):
        super().__init__()
        self.height = self.width = 8
        self.ouput_size = 8 * 8
        self.dropout_late = 0.5
        in_channels = 2
        channels = 16

        self.relu = nn.ReLU()
        self.layer1 = nn.Sequential(
            nn.Conv2d(in_channels, channels, 3, padding=1),
            nn.BatchNorm2d(channels),
            self.relu,
            nn.Conv2d(channels, channels, 3, padding=1),
            nn.BatchNorm2d(channels),
            self.relu,
            nn.Conv2d(channels, channels, 3),
            nn.BatchNorm2d(channels),
            self.relu,
            nn.Conv2d(channels, channels, 3),
            nn.BatchNorm2d(channels),
            self.relu,
        )

        self.fc_input = channels * (self.width - 4) * (self.height - 4)
        self.dropout = nn.Dropout(self.dropout_late, inplace=True)
        self.layer2 = nn.Sequential(
            nn.Linear(self.fc_input, 512),
            nn.BatchNorm1d(512),
            self.relu,
            self.dropout,
            nn.Linear(512, 256),
            nn.BatchNorm1d(256),
            self.relu,
            self.dropout,
        )

        self.fc3 = nn.Linear(256, 64)
        self.fc4 = nn.Linear(256, 1)
        self.softmax = nn.LogSoftmax(dim=1)
        self.tanh = nn.Tanh()

    def forward(self, x):
        x = self.layer1(x)
        x = x.view(-1, self.fc_input)
        x = self.layer2(x)
        policy = self.fc3(x)
        value = self.fc4(x)
        return self.softmax(policy), self.tanh(value)

class Dataset:
    def __init__(self, states, policy, values) -> None:
        self.states = states
        self.policy = policy
        self.values = values

    def __len__(self):
        return len(self.values)

    def __getitem__(self, idx):
        return self.states[idx], self.policy[idx], self.values[idx]

class LightingModule(pl.LightningModule):
    def __init__(self) -> None:
        super().__init__()
        self.model = Model()
        self.loss_p = nn.BCEWithLogitsLoss()
        self.loss_v = nn.MSELoss()

    def forward(self, x):
        return self.model(x)

    def shared_step(self, batch):
        x, policy, value = batch
        p, v = self.model(x)
        return self.loss_p(policy, p) + self.loss_v(value, v.squeeze())

    def training_step(self, batch, batch_idx):
        loss = self.shared_step(batch)
        self.log('train_loss', loss)
        return loss

    def validation_step(self, batch, batch_idx):
        loss = self.shared_step(batch)
        self.log('val_loss', loss)

    def configure_optimizers(self):
        return torch.optim.SGD(self.parameters(), lr=0.01)

def main():
    data_path = Path(sys.argv[1])
    policy = np.load(data_path / 'policy.npy')
    states = np.load(data_path / 'states.npy').astype(np.float32)
    values = np.load(data_path / 'values.npy').astype(np.float32)
    # print(policy.shape, states.shape, values.shape)
    train_p, val_p, train_s, val_s, train_v, val_v = train_test_split(
        policy, states, values, test_size=0.2, shuffle=True, random_state=42
    )
    train_dataset = Dataset(train_s, train_p, train_v)
    val_dataset = Dataset(val_s, val_p, val_v)
    train_dataloder = DataLoader( train_dataset, batch_size=256, shuffle=True)
    val_dataloder = DataLoader(val_dataset, batch_size=256)
    module = LightingModule()
    trainer = pl.Trainer(
        max_epochs=500,
        log_every_n_steps=10,
        callbacks=[
            ModelCheckpoint(monitor='val_loss'),
            EarlyStopping(monitor='val_loss'),
        ],
    )
    trainer.fit(module, train_dataloder, val_dataloder)
    dummy_input = torch.randn(1, 2, 8, 8)
    module.to_onnx("models/model.onnx", dummy_input, export_params=True)

if __name__ == '__main__':
    main()


import pandas as pd
import matplotlib.pyplot as plt
import matplotlib as mpl

df = pd.DataFrame([[500,450,520,610], [690,700,820,900],[1100,1030,1200,1380],
                    [1500,1650,1700,1850],[1990,2020,2300,2420],[1020,1600,2200,2550]],
                    index=[2015,2016,2017,2018,2019,2020],
                    columns=['1Q','2Q','3Q','4Q']
)

# df.to_csv('sales.csv',header=False,index=True,encoding='utf-8-sig')
# print(df)

font_name = mpl.font_manager.FontProperties(
    fname='c:/Windows/fonts/malgun.ttf').get_name()
mpl.rc('font',family=font_name)

# df를 그래프로 표현
df = df.transpose() #행,열 전환
df.plot()
plt.show()

# plt.figure(figsize=(7,6))
# plt.plot(df.columns,df.loc[2015],label=2015)
# plt.plot(df.columns,df.loc[2016],label=2016)
# plt.plot(df.columns,df.loc[2017],label=2017)
# plt.plot(df.columns,df.loc[2018],label=2018)
# plt.plot(df.columns,df.loc[2019],label=2019)
# plt.plot(df.columns,df.loc[2020],label=2020)

# plt.title('sales')
# plt.xlabel('Quarters')
# plt.ylabel('Sales')
# # plt.xticks(df.columns,df.columns,fontsize=10)
# plt.legend()
# plt.show()


#!/usr/bin/env python

import requests #for http requests
import re
import json

BASSINPREFIX = 'href="./niv2-bassin.php?CdEntVigiCru='
STATIONPREFIX = 'niv3-station.php?'

EXTRACTSTATIONIDRE = re.compile('CdStationHydro=([A-Z0-9]+)')

def extract_stationid(url):
    return EXTRACTSTATIONIDRE.findall(url)[0]

def parse_page(t,prefix):
    i=0
    while i!=-1:
        i = t.find(prefix,i)
        if i==-1:
            break
        j = t.find('"', i+len(prefix))
        yield t[i:j]
        i = j

def main():
    all_stations=[]
    r = requests.get('http://www.vigicrues.gouv.fr')
    t = r.text.encode(r.encoding)
    bassins = map(lambda hrefurl: int(hrefurl[len(BASSINPREFIX):]),list(set(parse_page(t,BASSINPREFIX))))
    for bassin in bassins:
        print('Bassin %s' % bassin)
        r = requests.get('http://www.vigicrues.gouv.fr/niv2-bassin.php?CdEntVigiCru=%s'%bassin)
        t = r.text.encode(r.encoding)
        all_stations += list(map(extract_stationid,parse_page(t,STATIONPREFIX)))
    all_stations = list(set(all_stations))
    all_stations_obj = []
    for station in all_stations:
        r = requests.get('http://www.vigicrues.gouv.fr/services/station.json/index.php?CdStationHydro=%s'%station)
        all_stations_obj.append({'id':station,'river':r.json()['LbCoursEau'].encode(r.encoding),'name':r.json()['LbStationHydro'].encode(r.encoding)})
    json.dump(all_stations_obj,open('stations_vigicrues.json','w'))

if __name__=='__main__':
    main()


from api_python.database.db import get_db

import sqlite3
import pytest

def test_get_db(app):
    # Assert the connection is the same within the same request context
    with app.app_context():
        conn = get_db()
        assert conn is get_db()

    # Assert the connection is closed
    with pytest.raises(sqlite3.ProgrammingError) as e:
        conn.execute('SELECT 1')
    assert 'closed' in str(e.value)

def test_init_db_command(runner, monkeypatch):
    class Recorder():
        called = False

    def fake_init_db():
        Recorder.called = True

    monkeypatch.setattr('api_python.database.db.init_db', fake_init_db)
    result = runner.invoke(args=['init-db'])
    assert 'Initialized' in result.output
    assert Recorder.called

def test_add_user_command(runner):
    result = runner.invoke(args=['add-user', 'example@ex.ex', '1234', 'user'])
    assert 'added' in result.output


#!/usr/bin/env python3
import configparser
import datetime
import json
import os
import subprocess
import sys

import requests
from dateutil.parser import parse

cache = None

def get_access_token(rclone_config_path=None, remote_name='onedrive'):
    global cache
    if cache is not None:
        return cache
    if rclone_config_path is None:
        rclone_config_path = os.path.expanduser('~/.config/rclone/rclone.conf')
    try:
        config = configparser.ConfigParser()
        config.read(rclone_config_path)
        token_info = json.loads(config[remote_name]['token'])
        if is_token_expired(token_info):
            raise Exception()
    except Exception as e:
        refresh_token_with_rclone(rclone_config_path, remote_name)
        return get_access_token(rclone_config_path, remote_name)
    cache = token_info['access_token']
    return cache

def is_rclone_installed():
    try:
        subprocess.run(['rclone', '--version'], check=True, stdout=subprocess.PIPE)
        return True
    except Exception as e:
        return False

def is_token_expired(token_info):
    expiry = parse(token_info['expiry'])
    now = datetime.datetime.now(datetime.timezone.utc)
    return expiry < now

def refresh_token_with_rclone(rclone_config_path, remote_name):
    if not is_rclone_installed():
        print('rclone is not installed. Please install it.', file=sys.stderr)
        sys.exit(1)
    try:
        subprocess.run(
            ['rclone', 'config', 'update', remote_name, 'token', rclone_config_path],
            check=True,
            stdout=subprocess.PIPE
        )
    except subprocess.CalledProcessError as e:
        subprocess.run(
            ['rclone', 'config', 'create', remote_name, 'onedrive', 'drive_type', 'personal', 'token', rclone_config_path],
            check=True,
            stdout=subprocess.PIPE
        )

def upload_file_to_onedrive(filepath, remotepath):
    if remotepath.startswith('/'):
        remotepath = remotepath[1:]
    if remotepath.endswith('/'):
        remotepath = remotepath[:-1]
    filename = os.path.basename(filepath)
    access_token = get_access_token()
    headers = {"Authorization": "Bearer " + access_token}
    endpoint = f"https://graph.microsoft.com/v1.0/me/drive/root:/{remotepath}/" + filename + ":/content"

    with open(filepath, 'rb') as f:
        r = requests.put(endpoint, headers=headers, data=f)
        r.raise_for_status()

    return r.json().get('id')

def create_embed_link(file_id):
    access_token = get_access_token()
    headers = {"Authorization": "Bearer " + access_token, "Content-Type": "application/json"}
    endpoint = "https://graph.microsoft.com/v1.0/me/drive/items/" + file_id + "/createLink"

    payload = {
        "type": "embed"
    }

    r = requests.post(endpoint, headers=headers, json=payload)
    r.raise_for_status()

    return r.json().get('link').get('webUrl')

def main(filepath, remotepath):
    file_id = upload_file_to_onedrive(filepath, remotepath)
    embed_link = create_embed_link(file_id)
    print(embed_link)

if __name__ == "__main__":
    if len(sys.argv) < 2:
        print("odpd: OneDrive Picture Direct Link Generator")
        print("  Usage: odpd.py <filepath> [remotepath]")
        sys.exit(1)
    main(sys.argv[1], "files/" if len(sys.argv) < 3 else sys.argv[2])


""" MISR-GRU for multi-image super resolution """

import torch.nn as nn
import torch
from torch.autograd import Variable

class ResidualBlock(nn.Module):
    def __init__(self, channel_size=64, kernel_size=3):
        '''
        Args:
            channel_size : int (c), number of hidden channels
            kernel_size : int, shape of a 2D kernel
        '''

        super(ResidualBlock, self).__init__()
        padding = kernel_size // 2
        self.block = nn.Sequential(
            nn.Conv2d(in_channels=channel_size, out_channels=channel_size, kernel_size=kernel_size, padding=padding),
            nn.PReLU(),
            nn.Conv2d(in_channels=channel_size, out_channels=channel_size, kernel_size=kernel_size, padding=padding),
            nn.PReLU()
        )

    def forward(self, x):
        '''
        Args:
            x : tensor (b, c, w, h), hidden representations
        Returns:
            x + residual: tensor (b, c, w, h), new hidden representations
        '''

        residual = self.block(x)
        return x + residual

class EncoderUnit(nn.Module):
    def __init__(self, in_channels, num_res_layers, kernel_size, channel_size):
        '''

        Each unit in the encoder is composed 2 conv layers and 2 residual blocks and parametric-relu as the
            activation function
        Args:
            in_channels : int (c_in), number of input channels
            num_res_blocks : int, number of resnet blocks
            kernel_size : int, shape of a 2D kernel,
            channel_size : int (c), number of hidden channels
        '''
        super(EncoderUnit, self).__init__()
        padding = kernel_size // 2
        self.init_layer = nn.Sequential(
            nn.Conv2d(in_channels=in_channels, out_channels=channel_size, kernel_size=kernel_size, padding=padding),
            nn.PReLU())

        res_layers = [ResidualBlock(channel_size, kernel_size) for _ in range(num_res_layers)]
        self.res_layers = nn.Sequential(*res_layers)

        self.final = nn.Sequential(
            nn.Conv2d(in_channels=channel_size, out_channels=channel_size, kernel_size=kernel_size, padding=padding)
        )

    def forward(self, x):
        '''
        Encodes an input tensor x.
        Args:
            x : tensor (b, c_in, W, H), input
        Returns:
            out: tensor (b, C, W, H), hidden representations
        '''

        x = self.init_layer(x)
        x = self.res_layers(x)
        x = self.final(x)
        return x

class ConvGRUUnit(nn.Module):
    def __init__(self, in_channels, hidden_channels, kernel_size):
        '''
            Args:
                in_channels : int (i_c), number of input channels
                hidden_channels : int (h_c), number of hidden channels
                kernel_size : kernel size
        '''
        super(ConvGRUUnit, self).__init__()
        padding = kernel_size // 2
        self.hidden_channels = hidden_channels
        self.reset_gate = nn.Conv2d(in_channels + hidden_channels, hidden_channels, kernel_size, padding=padding)
        self.update_gate = nn.Conv2d(in_channels + hidden_channels, hidden_channels, kernel_size, padding=padding)
        self.out_gate = nn.Conv2d(in_channels + hidden_channels, hidden_channels, kernel_size, padding=padding)

    def forward(self, x, prev_state):
        '''
            Args:
                x : tensor (b, c, w, h), input to gru unit
            Returns:
                out: tensor (b, h_c, w, h), hidden states
        '''

        # generate empty prev_state, if it is None
        if prev_state is None:
            state_size = [x.shape[0], self.hidden_channels] + list(x.shape[2:])
            prev_state = Variable(torch.zeros(state_size))
            if torch.cuda.is_available():
                prev_state = prev_state.cuda()

        stacked = torch.cat([x, prev_state], dim=1)
        update = torch.sigmoid(self.update_gate(stacked))
        reset = torch.sigmoid(self.reset_gate(stacked))
        candidate_state = torch.tanh(self.out_gate(torch.cat([x, prev_state * reset], dim=1)))
        new_state = (1 - update) * prev_state + update * candidate_state

        return new_state

class FusionModule(nn.Module):
    def __init__(self, fuse_config):
        '''
        Args:
            fuse_config : dict, fusion configuration
        '''
        super(FusionModule, self).__init__()
        self.input_channels = fuse_config["in_channels"]
        self.num_hidden_layers = fuse_config["num_hidden_layers"]
        hidden_channels = fuse_config["hidden_channels"]
        kernel_sizes = fuse_config["kernel_sizes"]

        gru_units = []
        for i in range(0, self.num_hidden_layers):
            cur_input_dim = self.input_channels if i == 0 else hidden_channels[i - 1]
            gru_unit = ConvGRUUnit(in_channels=cur_input_dim, hidden_channels=hidden_channels[i],
                                   kernel_size=kernel_sizes[i])
            gru_units.append(gru_unit)
        self.gru_units = nn.ModuleList(gru_units)

    def forward(self, x, alphas, h=None):

        '''
            Args:
                x : tensor (B, L, C, W, H), hidden states
                alphas : tensor (B, L, 1, 1, 1), boolean indicator (0 if padded low-res view, 1 otherwise)
            Returns:
                out: tensor (B, C, W, H), fused hidden state
        '''

        if h is None:
            hidden_states = [None] * self.num_hidden_layers
        num_low_res = x.shape[1]
        cur_layer_input = x

        for l in range(self.num_hidden_layers):
            gru_unit = self.gru_units[l]
            h = hidden_states[l]

            out = []
            for t in range(num_low_res):
                h = gru_unit(cur_layer_input[:, t, :, :, :], h)
                out.append(h)

            out = torch.stack(out, dim=1)
            cur_layer_input = out

        fused_representations = torch.sum(cur_layer_input * alphas, 1) / torch.sum(alphas, 1)

        return fused_representations

class Decoder(nn.Module):
    def __init__(self, dec_config):
        '''
        Args:
            dec_config : dict, decoder configuration
        '''

        super(Decoder, self).__init__()

        self.deconv = nn.Sequential(nn.ConvTranspose2d(in_channels=dec_config["deconv"]["in_channels"],
                                                       out_channels=dec_config["deconv"]["out_channels"],
                                                       kernel_size=dec_config["deconv"]["kernel_size"],
                                                       stride=dec_config["deconv"]["stride"]),
                                    nn.PReLU())

        self.final = nn.Conv2d(in_channels=dec_config["final"]["in_channels"],
                               out_channels=dec_config["final"]["out_channels"],
                               kernel_size=dec_config["final"]["kernel_size"],
                               padding=dec_config["final"]["kernel_size"] // 2)

    def forward(self, x):
        '''
        Decodes a hidden state x.
        Args:
            x : tensor (b, c, w, h), fused hidden state
        Returns:
            out: tensor (b, c_out, 3xw, 3xw), super resolved image

        '''
        x = self.deconv(x)
        x = self.final(x)

        return x

class MISRGRU(nn.Module):
    ''' MISRGRU, a neural network for multi-image super resolution (MISR) using ConvGRU. '''

    def __init__(self, config):
        '''
        Args:
            config : dict, configuration file
        '''

        super(MISRGRU, self).__init__()
        self.unit1 = EncoderUnit(in_channels=config["encoder"]["in_channels"],
                                 num_res_layers=config["encoder"]["num_res_blocks"],
                                 kernel_size=config["encoder"]["kernel_size"],
                                 channel_size=config["encoder"]["channel_size"])
        self.fuse = FusionModule(config["fusion"])
        self.decode = Decoder(config["decoder"])
        self.unit2 = EncoderUnit(in_channels=config["encoder"]["channel_size"] * 2,
                                 num_res_layers=config["encoder"]["num_res_blocks"],
                                 kernel_size=config["encoder"]["kernel_size"],
                                 channel_size=config["encoder"]["channel_size"])

    def forward(self, lrs, alphas):
        '''
        Super resolves a batch of low-resolution images.
        Args:
            lrs : tensor (B, L, W, H), low-resolution images
            alphas : tensor (B, L), boolean indicator (0 if padded low-res view, 1 otherwise)
        Returns:
            srs: tensor (B, C_out, W, H), super-resolved images
        '''

        ###################### Encode #########################
        batch_size, num_low_res, height, width = lrs.shape
        # Extend channel dimension
        lrs = lrs.view(batch_size, num_low_res, 1, height, width)
        # create a reference view based on median statistics
        refs, _ = torch.median(lrs[:, :9], 1)

        alphas = alphas.view(-1, num_low_res, 1, 1, 1)

        # encode LR views
        lrs = lrs.view(batch_size * num_low_res, 1, height, width)
        lrs = self.unit1(lrs)
        lrs = lrs.view(batch_size, num_low_res, -1, height, width)

        # encode ref view
        refs = self.unit1(refs)
        refs = refs.unsqueeze(1)
        refs = refs.repeat(1, num_low_res, 1, 1, 1)

        # Co-register ref encoded features with LR encoded features
        out = torch.cat([lrs, refs], 2)
        out = out.view(batch_size * num_low_res, -1, height, width)
        out = self.unit2(out)
        out = out.view(batch_size, num_low_res, -1, height, width)  # tensor (b, l, c, w, h)

        ###################### Fuse #########################
        out = self.fuse(out, alphas)

        ###################### Decode #########################
        out = self.decode(out)

        return out


from captcha.image import ImageCaptcha
from random import randint, randbytes
from hashlib import sha1, sha256
from os.path import isfile

def create_captcha_salt():
    print("Creating captcha salt...")
    f = open("./data/captcha-salt.txt", "x")
    f.write(sha256(randbytes(1024)).hexdigest())
    f.close()

def read_captcha_salt():
    if not isfile("./data/captcha-salt.txt"):
        create_captcha_salt()

    f = open("./data/captcha-salt.txt", "r")
    s = f.read()
    f.close()
    return s

capctha_salt = read_captcha_salt()

img_captcha = ImageCaptcha(
    width=160,
    height=80,
    fonts=["./captchas/fonts/Caveat.ttf", "./captchas/fonts/IndieFlower.ttf"]
)

def generate_captcha_pair():
    numbers = "".join([str(randint(0, 9)) for _ in range(4)])
    hash = sha1()
    hash.update(numbers.encode("ascii"))
    hash.update(capctha_salt.encode("ascii"))

    return (numbers, hash.hexdigest())

def generate_captcha():
    (num, hash) = generate_captcha_pair()
    img_captcha.write(num, f"./data/captchas/{hash}.png", "png")
    return hash

def validate_captcha_pair(numbers: str, hash: str):
    h = sha1()
    h.update(numbers.encode("ascii"))
    h.update(capctha_salt.encode("ascii"))
    valid = hash == h.hexdigest()

    return valid


from corpus.datasources.webscrape import WebScrape
from corpus.event import EventStorage,EventManager, EventSeriesManager
import datetime
from enum import Enum
import glob
import re
import os
import sys
import threading
import time
from argparse import ArgumentParser
from argparse import RawDescriptionHelpFormatter
#from lodstorage.jsonpicklemixin import JsonPickleMixin
from lodstorage.storageconfig import StorageConfig
import corpus.datasources.wikicfp as wcfp
#import jsonpickle

class CrawlType(Enum):
    '''
    possible supported storage modes
    '''
    EVENT = "Event"
    SERIES = "Series"

    @property
    def urlPrefix(self):
        baseUrl="http://www.wikicfp.com/cfp"
        if self.value is CrawlType.EVENT.value:
            url= f"{baseUrl}/servlet/event.showcfp?eventid="
        elif self.value is CrawlType.SERIES.value:
            url= f"{baseUrl}/program?id="
        return url

    @classmethod
    def isValid(cls,value:str)->bool:
        '''
        check whether the given value is valid

        Args:
            value(str): the value to check

        Return:
            bool: True if the value is a valid value of this enum
        '''
        return value in ["Event","Series"]

    @classmethod
    def ofValue(cls,value:str):
        if value=="Event":
            return CrawlType.EVENT
        elif value=="Series":
            return CrawlType.SERIES
        else:
            return None

class CrawlBatch(object):
    '''
    describe a batch of pages to fetch metadata from
    '''

    def __init__(self,threads:int,startId:int,stopId:int,crawlTypeValue:str,threadIndex=None,):
        '''
        construct me with the given number of threads, startId and stopId

        Args:
           threads(int): number of threads to use
           startId(int): id of the event to start crawling with
           stopId(int): id of the event to stop crawling
           crawlTypeValue(str): the type of crawling (Event or Series)
           threadIndex(int): the index of this batch
        '''
        self.threads=threads
        self.threadIndex=threadIndex
        self.startId=startId
        self.stopId=stopId
        if startId <= stopId:
            self.step = +1
            self.total = stopId-startId+1
        else:
            self.step = -1
            self.total = startId-stopId+1
        self.batchSize = self.total // threads
        if not CrawlType.isValid(crawlTypeValue):
            raise Exception(f"Invalid crawlType {crawlTypeValue}")
        self.crawlType=CrawlType.ofValue(crawlTypeValue)

    def split(self)->list:
        '''
        split me for my threads
        '''
        crawlBatches=[]
        for threadIndex in range(self.threads):
            s = self.startId + threadIndex * self.batchSize
            e = s + self.batchSize-1
            splitBatch=CrawlBatch(1,s, e,self.crawlType.value,threadIndex)
            crawlBatches.append(splitBatch)
        return crawlBatches

    def __str__(self):
        '''
        get my string representation
        Return
            str: my string representation
        '''
        text=f"WikiCFP {self.crawlType.value} IDs {self.startId} - {self.stopId} ({self.threads} threads of {self.batchSize} IDs each)"
        return text

class WikiCfpScrape(object):
    '''
    support events from http://www.wikicfp.com/cfp/
    '''

    def __init__(self,jsonEventManager,jsonEventSeriesManager,profile:bool=True,debug:bool=False,jsondir:str=None,limit=200000,batchSize=1000,showProgress=True):
        '''
        Constructor

        Args:
            config(StorageConfig): the storage configuration to use
            debug(bool): if True debug for crawling is on
            limit(int): maximum number of entries to be crawled
            batchSize(int): default size of batches
            showProgress(bool): if True show Progress
        '''
        self.debug=debug
        self.limit=limit
        self.batchSize=batchSize
        self.showProgress=showProgress
        self.jsonEventManager=jsonEventManager
        self.jsonEventSeriesManager=jsonEventSeriesManager
        self.jsonManagers={
            "Event": jsonEventManager,
            "Series": jsonEventSeriesManager
        }
        self.profile=profile
        if jsondir is not None:
            self.jsondir=jsondir
        else:
            cachePath=self.jsonEventManager.config.getCachePath()
            self.jsondir=f"{cachePath}/wikicfp"
            if not os.path.exists(self.jsondir):
                    os.makedirs(self.jsondir)

    def getManager(self,crawlType:CrawlType):
        '''
        get the manager for the given crawlType

        Args:
           crawlType(CrawlType) the manager for the given CrawlType
        '''
        manager=self.jsonManagers[crawlType.value]
        return manager

    def cacheToJsonManager(self,crawlType:CrawlType):
        '''
        cache the crawlType entities to my json manager for that crawlType

        Args:
            crawlType(CrawlType): the CrawlType to work with

        Return:
            EntityBasemanager: the entityManager

        '''
        jsonEm=self.getManager(crawlType)
        if jsonEm.isCached():
            jsonEm.fromStore()
        else:
            self.crawlFilesToJson(crawlType=crawlType,withStore=True)
        return jsonEm

    def crawlFilesToJson(self,crawlType:CrawlType,withStore=True):
        '''
        convert the crawlFiles to Json

        Args:
            crawlType(CrawlType): the CrawlType to work with
            withStore(bool): True if the data should be stored after collecting

        Return:
            jsonEm(EventBaseManager): the JSON - storage based EventBaseManager to use to collect the results
        '''
        # crawling is not done on startup but need to be done
        # in command line mode ... we just collect the json crawl result files here
        #self.crawl(startId=startId,stopId=stopId)
        jsonEm=self.getManager(crawlType)
        entityList=jsonEm.getList()
        startTime=time.time()
        jsonFiles=self.jsonFiles(crawlType)
        if len(jsonFiles)==0:
            if self.profile or self.debug:
                print(f"No wikiCFP crawl json backups for {crawlType.value} available")
        else:
            # loop over all crawled files
            for jsonFilePath in jsonFiles:
                config=StorageConfig.getJSON(debug=self.debug)
                config.cacheFile=jsonFilePath
                if crawlType.value is CrawlType.EVENT.value:
                    batchEm=EventManager(name=jsonFilePath,clazz=jsonEm.clazz,config=config)
                elif crawlType.value is CrawlType.SERIES.value:
                    batchEm=EventSeriesManager(name=jsonFilePath,clazz=jsonEm.clazz,config=config)
                # legacy mode -file were created before ORM Mode
                # was available - TODO: make new files available in ORM mode with jsonable
                #jsonPickleEm=JsonPickleMixin.readJsonPickle(jsonFileName=jsonFilePath,extension='.json')
                #jsonPickleEvents=jsonPickleEm['events']
                #if self.showProgress:
                #    print("%4d: %s" % (len(jsonPickleEvents),jsonFilePath))
                batchEm.fromStore(cacheFile=jsonFilePath)
                for entity in batchEm.getList():
                    if hasattr(entity, "startDate"):
                        if entity.startDate is not None:
                            entity.year=entity.startDate.year
                    doAdd=not(entity.deleted)
                    # SPAM Filter
                    if hasattr(entity, "locality"):
                        if entity.locality.startswith("1"):
                            doAdd=False
                    # Series Filter
                    if hasattr(entity, "title"):
                        if entity.title.startswith("WikiCFP : Call For Papers of Conferences, Workshops and Journals"):
                            doAdd=False

                    if doAdd:
                        entityList.append(entity)
            if self.profile:
                elapsed=time.time()-startTime
                print (f"read {len(entityList)} {crawlType.value} records in {elapsed:5.1f} s")
            if withStore:
                jsonEm.store(limit=self.limit,batchSize=self.batchSize)
        return jsonEm

    def jsonFiles(self,crawlType:CrawlType)->list:
        '''
        get the list of the json files that have my data

        Args:
            crawlType(CrawlType): the tpe of the files

        Return:
            list: a list of json file names
        '''
        jsonFiles=sorted(glob.glob(f"{self.jsondir}/wikicfp_{crawlType.value}*.json"),key=lambda path:int(re.findall(r'\d+',path)[0]))
        return jsonFiles

    def getJsonFileName(self,crawlBatch):
        '''
        get my the JsonFileName

        Args:
            crawlBatch(CrawlBatch): the batch to crawl):

        Return:
            str: the json file name for this batch
        '''
        jsonFilePath=f"{self.jsondir}/wikicfp_{crawlBatch.crawlType.value}{crawlBatch.startId:06d}-{crawlBatch.stopId:06d}.json"
        return jsonFilePath

    def getBatchEntityManager(self,crawlBatch:CrawlBatch):
        '''
        get the batch Entity Manager for the given crawlBatch

        Args:
            crawlBatch(CrawlBatch): the batch to crawl):

        Return:
            EntityManager: either a Event or a Series Manager
        '''
        jsonFilepath=self.getJsonFileName(crawlBatch)
        config=EventStorage.getStorageConfig(debug=self.debug, mode="json")
        config.cacheFile=jsonFilepath
        crawlType=crawlBatch.crawlType
        print(f"CrawlBatch has crawlType {type(crawlType)}{crawlType}/{crawlType.value}")
        if crawlType.value is CrawlType.EVENT.value:
            batchEm=wcfp.WikiCfpEventManager(config=config)
        elif crawlType.value is CrawlType.SERIES.value:
            batchEm=wcfp.WikiCfpEventSeriesManager(config=config)
        else:
            raise Exception(f"Invalid crawlType {crawlType}")
        return batchEm

    def crawl(self,crawlBatch:CrawlBatch):
        '''
        see https://github.com/TIBHannover/confIDent-dataScraping/blob/master/wikicfp.py

        Args:
            crawlBatch(CrawlBatch): the batch to crawl
        '''

        print(f'crawling {crawlBatch}')
        batchEm=self.getBatchEntityManager(crawlBatch)

        # get all ids
        crawlType=crawlBatch.crawlType
        for eventId in range(int(crawlBatch.startId), int(crawlBatch.stopId+1), crawlBatch.step):
            wEvent=WikiCfpEventFetcher(crawlType=crawlType)
            retry=1
            maxRetries=3
            retrievedResult=False
            while not retrievedResult:
                try:
                    rawEvent=wEvent.fromEventId(eventId)
                    if crawlType.value is CrawlType.EVENT.value:
                        event=wcfp.WikiCfpEvent()
                        event.fromDict(rawEvent)
                        title="? deleted: %r" %event.deleted if not 'title' in rawEvent else event.title
                        batchEm.getList().append(event)
                    elif crawlType.value is CrawlType.SERIES.value:
                        eventSeries=wcfp.WikiCfpEventSeries()
                        eventSeries.fromDict(rawEvent)
                        title="?" if not 'title' in rawEvent else eventSeries.title
                        batchEm.getList().append(eventSeries)
                    retrievedResult=True
                except Exception as ex:
                    if "HTTP Error 500" in str(ex):
                        print(f"{eventId} inaccessible due to HTTP Error 500")
                        retrievedResult=True
                    elif "timed out" in str(ex):
                        print(f"{eventId} access timed Out on retry attempt {retry}")
                        retry+=1
                        if retry>maxRetries:
                            raise ex
                    else:
                        raise ex
                    pass

            print(f"{eventId:06d}: {title}")

        batchEm.store()
        return batchEm

    def threadedCrawl(self,crawlBatch:CrawlBatch):
        '''
        crawl with the given number of threads, startId and stopId

        Args:
            crawlBatch(CrawlBatch): the batch to crawl
        '''
        # determine the eventId range for each threaded job
        startTime=time.time()

        msg=f'Crawling {crawlBatch}'
        print(msg)

        # this list will contain all threads -> we can wait for all to finish at the end
        jobs = []

        # now start each thread with its id range and own filename
        for crawlBatch in crawlBatch.split():

            thread = threading.Thread(target = self.crawl, args=(crawlBatch,))
            jobs.append(thread)

        for job in jobs:
            job.start()

        # wait till all threads have finished before print the last output
        for job in jobs:
            job.join()

        if self.debug:
            elapsed=time.time()-startTime
            print(f'crawling done after {elapsed:5.1f} s')

class WikiCfpEventFetcher(object):
    '''
    a single WikiCfpEentFetcher to fetch and event or series
    '''
    def __init__(self,crawlType=CrawlType.EVENT,debug=False,showProgress:bool=True,timeout=20):
        '''
        construct me

        Args:
            showProgress(bool): if True show progress
            timeout(float): the default timeout

        '''
        self.debug=debug
        self.crawlType=crawlType
        self.showProgress=showProgress
        self.progressCount=0
        self.timeout=timeout

    def fromTriples(self,rawEvent,triples):
        '''
        get the rawEvent dict from the given triple e.g.:

        v:Event(v:summary)=IDC 2009
        v:Event(v:eventType)=Conference
        v:Event(v:startDate)=2009-06-03T00:00:00
        v:Event(v:endDate)=2009-06-05T23:59:59
        v:Event(v:locality)=Milano, Como, Italy
        v:Event(v:description)= IDC  2009 : The 8th International Conference on Interaction Design and Children
        v:Address(v:locality)=Milano, Como, Italy
        v:Event(v:summary)=Submission Deadline
        v:Event(v:startDate)=2009-01-19T00:00:00
        v:Event(v:summary)=Notification Due
        v:Event(v:startDate)=2009-02-20T00:00:00
        v:Event(v:summary)=Final Version Due
        v:Event(v:startDate)=2009-03-16T00:00:00
        '''
        recentSummary=None
        for s,p,o in triples:
            s=s.replace("v:","")
            p=p.replace("v:","")
            if self.debug:
                print ("%s(%s)=%s" % (s,p,o))
            if recentSummary in ['Submission Deadline','Notification Due','Final Version Due']:
                key=recentSummary.replace(" ","_")
            else:
                key=p
            if p.endswith('Date'):
                dateStr=o
                if dateStr=="TBD":
                    o=None
                else:
                    o=datetime.datetime.strptime(
                        dateStr, "%Y-%m-%dT%H:%M:%S").date()
            if not key in rawEvent:
                rawEvent[key]=o
            if p=="summary":
                recentSummary=o
            else:
                recentSummary=None

    @staticmethod
    def getUrl(cfpid,crawlType:CrawlType=CrawlType.EVENT)->str:
        '''
        Args:
            cfpid(int): the WikiCFP id of the event or series
            crawlType(CrawlType): Event or Series

        Returns:
            the WikiCfP url
        '''
        url=f"{crawlType.urlPrefix}{cfpid}"
        return url

    @classmethod
    def getLatestEvent(cls,debug=False,showProgress=True):
        '''
        get the latest Event doing a binary search
        '''
        wikicfp=WikiCfpEventFetcher(debug,showProgress=showProgress)
        wikicfp.progressCount=0
        wikicfp.getLatesEvetFromPair()

    def getHighestNonDeletedIdInRange(self,fromId:int,toId:int)->int:
        '''
        get the event with the highest id int the range fromId to toId that is not deleted

        Args:
            fromId(int): minimum id to search from
            toId(int): maximum id to search to

        Returns:
            int: maxium id of an event that is not deleted or None if there is none in this range
        '''
        maxId=None
        for eventId in range(fromId,toId+1):
            if self.showProgress:
                print(".",end='',flush=True)
                self.progressCount+=1
                if self.progressCount % 50 == 0:
                    print(flush=True)
            rawEvent=self.fromEventId(eventId)
            if not rawEvent['deleted']:
                maxId=eventId
        return maxId

    def getLatesEvetFromPair(self,low=5000,high=300000,margin=40):
        '''
        get the latest Event doing a binary search

        Args:
            low(int): lower index to search from
            hight(int): upper index boundary
        '''
        if high-low>margin+1:
            mid=(high+low)//2
            midId=(self.getHighestNonDeletedIdInRange(mid-margin, mid))
            if midId:
                return self.getLatesEvetFromPair(mid+1,high)
            else:
                return self.getLatesEvetFromPair(low, mid-1)
        else:
            return mid
        pass

    def fromEventId(self,cfpid:int):
        '''
        see e.g. https://github.com/andreeaiana/graph_confrec/blob/master/src/data/WikiCFPCrawler.py

        Args:
            cfpid(int): the wikicfp id to use
        '''
        url=WikiCfpEventFetcher.getUrl(cfpid,self.crawlType)
        return self.fromUrl(url)

    def rawEventFromWebScrape(self,rawEvent:dict,triples:list,scrape:WebScrape):
        '''
        fill the given rawEvent with the data derived from the scrape

        Args:
            rawEvent(dict): the event dictionary
            triples(list): the triples found
            scrape(WebScrape): the webscrape object to be used for parsing
        '''
        if len(triples)==0:
            #scrape.printPrettyHtml(scrape.soup)
            firstH3=scrape.fromTag(scrape.soup, 'h3')
            if "This item has been deleted" in firstH3:
                rawEvent['deleted']=True
        else:
            self.fromTriples(rawEvent,triples)
            # add series information
            # Tag: International Semantic Web Conference
            m,seriesText=scrape.findLinkForRegexp(r'/cfp/program\?id=([0-9]+).*')
            if m:
                seriesId=m.group(1)
                rawEvent['seriesId']=seriesId
                rawEvent['series']=seriesText
                pass

            if 'summary' in rawEvent:
                rawEvent['acronym']=rawEvent.pop('summary').strip()
            if 'description' in rawEvent:
                rawEvent['title']=rawEvent.pop('description').strip()

    def rawEventSeriesFromWebScrape(self,rawEvent:dict,scrape:WebScrape):
        '''
        fill the given rawEventSeries with the data derived from the scrape

        Args:
            rawEvent(dict): the event dictionary
            scrape(WebScrape): the webscrape object to be used for parsing
        '''
        title=scrape.soup.find("title")
        rawEvent["title"]=title.text.strip()
        dblpM,_text=scrape.findLinkForRegexp(r'http://dblp.uni-trier.de/db/([a-zA-Z0-9/-]+)/index.html')
        if dblpM:
            dblpSeriesId=dblpM.group(1)
            rawEvent['dblpSeriesId']=dblpSeriesId
        pass

    def fromUrl(self,url:str)->dict:
        '''
        get the event form the given url

        Args:
            url(str): the url to get the event from

        Returns:
            dict: a raw event dict or None if an error occured

        '''
        regexp=r"^"+self.crawlType.urlPrefix.replace("?","\?")+"(\d+)$"
        m=re.match(regexp,url)
        if not m:
            raise Exception("Invalid URL %s" % (url))
        else:
            cfpId=int(m.group(1))
        rawEvent={}
        if self.crawlType.value is CrawlType.EVENT.value:
            rawEvent['eventId']=f"{cfpId}"
        elif self.crawlType.value is CrawlType.SERIES.value:
            rawEvent['seriesId']=f"{cfpId}"
        rawEvent['url']=url
        rawEvent['wikiCfpId']=cfpId
        rawEvent['deleted']=False
        scrape=WebScrape(debug=self.debug,timeout=self.timeout)
        triples=scrape.parseRDFa(url)
        if scrape.err:
            raise Exception(f"fromUrl {url} failed {scrape.err}")
        if self.crawlType.value is CrawlType.EVENT.value:
            self.rawEventFromWebScrape(rawEvent, triples, scrape)
        else:
            self.rawEventSeriesFromWebScrape(rawEvent,scrape)
        return rawEvent

__version__ = 0.4
__date__ = '2020-06-22'
__updated__ = '2021-08-09'

DEBUG = 1

def main(argv=None): # IGNORE:C0111
    '''main program.'''

    if argv is None:
        argv=sys.argv[1:]

    program_name = os.path.basename(__file__)
    program_version = "v%s" % __version__
    program_build_date = str(__updated__)
    program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date)
    program_shortdesc = "script to read event metadata from http://wikicfp.com"
    user_name="Wolfgang Fahl"
    program_license = '''%s

  Created by %s on %s.
  Copyright 2020-2021 TIB Hannover, Wolfgang Fahl. All rights reserved.

  Licensed under the Apache License 2.0
  http://www.apache.org/licenses/LICENSE-2.0

  Distributed on an "AS IS" basis without warranties
  or conditions of any kind, either express or implied.

USAGE
''' % (program_shortdesc,user_name, str(__date__))

    try:
        # Setup argument parser
        parser = ArgumentParser(description=program_license, formatter_class=RawDescriptionHelpFormatter)
        parser.add_argument("-d", "--debug", dest="debug", action="count", help="set debug level [default: %(default)s]")
        parser.add_argument('-V', '--version', action='version', version=program_version_message)
        parser.add_argument('--startId', type=int, help='eventId to start crawling from', required=True)
        parser.add_argument('--stopId', type=int, help='eventId to stop crawling at', required=True)
        parser.add_argument('--crawlType',type=str,default="Event",help="The crawlType - Event or Series")
        parser.add_argument('-p','--targetPath',type=str,help="targetPath (JSON directory) for crawl results")
        parser.add_argument('-t','--threads', type=int, help='number of threads to start', default=10)

        # Process arguments
        args = parser.parse_args(argv)
        wikiCfp=wcfp.WikiCfp()
        wikiCfpScrape=wikiCfp.wikiCfpScrape
        wikiCfpScrape.jsondir=args.targetPath
        wikiCfpScrape.debug=args.debug
        crawlBatch=CrawlBatch(args.threads, args.startId, args.stopId,args.crawlType,None)
        wikiCfpScrape.threadedCrawl(crawlBatch)

    except KeyboardInterrupt:
        ### handle keyboard interrupt ###
        return 1
    except Exception as e:
        if DEBUG:
            raise(e)
        indent = len(program_name) * " "
        sys.stderr.write(program_name + ": " + repr(e) + "\n")
        sys.stderr.write(indent + "  for help use --help")
        return 2

if __name__ == "__main__":
    if DEBUG:
        sys.argv.append("-d")
    sys.exit(main())


import json, csv
from argparse import ArgumentParser
from analysis import Motif, ensemble_from_string, load_ground_truth, prc

parser = ArgumentParser()
parser.add_argument("--ground-truth", required=True)
parser.add_argument("--image", required=True)
parser.add_argument("--output", required=True)
parser.add_argument("--ensemble", default="count")

args = parser.parse_args()

# main
if __name__ == "__main__":
    # load the ground truth
    with open(args.ground_truth, 'r') as f:
        gt, _ = load_ground_truth(json.load(f))

    # load the image
    motifs = []
    with open(args.image, 'r') as f:
        for motif in json.load(f):
            motifs.append(Motif.of_json(motif))

    # compute the ensemble
    ensemble = ensemble_from_string(args.ensemble)(motifs)

    # if there's also a prc output, do that thing
    prc(ensemble, gt, output=args.output)


# -*- coding: utf-8 -*-
# __author__ = 'zoujiaxun'
import os
import unittest
import pandas as pd
from Bio import SeqIO
import re

from biocluster.agent import Agent
from biocluster.tool import Tool
from mbio.packages.whole_transcriptome.utils import runcmd
from biocluster.core.exceptions import OptionError

class GetfastaCircAgent(Agent):
    '''
    last_modify: 2019.10.21
    '''

    def __init__(self, parent):
        super(GetfastaCircAgent, self).__init__(parent)
        options = [
            {'name': 'circbase', 'type': 'infile', 'format': 'whole_transcriptome.fasta'},
            {'name': 'circbase_bed', 'type': 'outfile', 'format': 'whole_transcriptome.common'},
            {'name': 'genome', 'type': 'infile', 'format': 'whole_transcriptome.fasta'},

            {'name': 'circfasta', 'type': 'outfile', 'format': 'ref_rna_v2.fasta'},

        ]
        self.add_option(options)

    def check_options(self):
        pass

    def set_resource(self):
        self._cpu = 10
        self._memory = '40G'

    def end(self):
        super(GetfastaCircAgent, self).end()

class GetfastaCircTool(Tool):
    def __init__(self, config):
        super(GetfastaCircTool, self).__init__(config)
        self.program = {
            'bedtools': 'bioinfo/rna/bedtools2-master/bin/bedtools'

        }
        self.file = {
            'circbase_bed':os.path.join(self.output_dir,'circbase_bed.bed'),
            'circfasta': os.path.join(self.output_dir, 'circrna.fasta'),

        }
        self.script = {

        }

    def run(self):
        super(GetfastaCircTool, self).run()
        self.run_circbase_bed()
        # self.run_bed()
        self.run_getfasta()
        self.set_output()
        self.end()

    def run_circbase_bed(self):
        with open(self.file['circbase_bed'], 'w') as bed:
            for seq_record in SeqIO.parse(self.option('circbase').path, "fasta"):
                id = seq_record.id
                (circbase_id, chr_start_end_strant, host_gene_id, protein_id) = id.rstrip().split('|')
                chr_start_end = chr_start_end_strant[:-1]
                (chr, start, end) = re.split('[:-]', chr_start_end)
                bed.write(chr+'\t'+str(int(start)+1)+'\t'+end+'\n')

    #
    #
    #
    # def run_bed(self):
    #     df_bed = dict()
    #     df = pd.read_table(self.option('circbase').path)
    #     df_bed['chr']=df['chr']
    #     df_bed['start']=df['circrna_start']
    #     df_bed['end']=df['circrna_end']
    #     circ_bed = pd.DataFrame(df_bed)
    #     circ_bed_reindex = circ_bed.reindex(columns=['chr', 'start', 'end'])
    #     circ_bed_reindex.to_csv(self.file['circbed'], index=False, header=False, sep='\t')

    def run_getfasta(self):
        cmd = '{} getfasta -fi {} -bed {} -fo {}'.format(self.program['bedtools'], self.option('genome').path, self.file['circbase_bed'], self.file['circfasta'])
        cmd_name = 'run_getfasta'
        runcmd(self,cmd_name,cmd)

    def set_output(self):
        self.option('circfasta').set_path(self.file['circfasta'])

class TestFunction(unittest.TestCase):
    '''
    This is test for the tool. Just run this script to do test.
    '''

    def test(self):
        import random
        from mbio.workflows.single import SingleWorkflow
        from biocluster.wsheet import Sheet
        data = {
            'id': 'getfasta_{}_{}'.format(random.randint(1000, 9999), random.randint(1000, 9999)),
            'type': 'tool',
            'name': 'whole_transcriptome.circrna.getfasta_circ',
            'instant': False,
            'options': {
                'genome': '/mnt/ilustre/users/sanger-dev/sg-users/qinjincheng/database/mouse/Mus_mm9.fasta',
                'circbase': '/mnt/ilustre/users/sanger-dev/sg-users/zoujiaxun/mouse_mm9_circRNAs_putative_spliced_sequence.fa'

            }
        }
        wsheet = Sheet(data=data)
        wf = SingleWorkflow(wsheet)
        wf.run()

if __name__ == '__main__':
    suite = unittest.TestSuite()
    suite.addTests([TestFunction('test')])
    unittest.TextTestRunner(verbosity=2).run(suite)

{'random': 'random', 'SingleWorkflow': 'mbio.workflows.single.SingleWorkflow', 'Sheet': 'biocluster.wsheet.Sheet'}


import traceback
from flask_restful import Resource,reqparse
from flask_jwt import jwt_required

from models.stats import StatsModel
from utils.performance import GamePerformance

class Stats(Resource):
    # (tournamentID, playerID, attendance, appearance, start, goal,
    #   penalty, penaltyShootout, penaltyTaken, ownGoal, header, weakFootGoal,
    #   assist, yellow, red, cleanSheet, penaltySaved)
    parser = reqparse.RequestParser()
    parser.add_argument('attendance', type=int, required=True, help="This field cannot be blank.")
    parser.add_argument('appearance', type=int, required=True, help="This field cannot be blank.")
    parser.add_argument('start', type=int, required=True, help="This field cannot be blank.")
    parser.add_argument('goal', type=int, required=True, help="This field cannot be blank.")
    parser.add_argument('penalty', type=int, required=True, help="This field cannot be blank.")
    parser.add_argument('freekick', type=int, required=True, help="This field cannot be blank.")
    parser.add_argument('penaltyShootout', type=int, required=True, help="This field cannot be blank.")
    parser.add_argument('penaltyTaken', type=int, required=True, help="This field cannot be blank.")
    parser.add_argument('ownGoal', type=int, required=True, help="This field cannot be blank.")
    parser.add_argument('header', type=int, required=True, help="This field cannot be blank.")
    parser.add_argument('weakFootGoal', type=int, required=True, help="This field cannot be blank.")
    parser.add_argument('otherGoal', type=int, required=True, help="This field cannot be blank.")
    parser.add_argument('assist', type=int, required=True, help="This field cannot be blank.")
    parser.add_argument('yellow', type=int, required=True, help="This field cannot be blank.")
    parser.add_argument('red', type=int, required=True, help="This field cannot be blank.")
    parser.add_argument('cleanSheet', type=int, required=True, help="This field cannot be blank.")
    parser.add_argument('penaltySaved', type=int, required=True, help="This field cannot be blank.")

    def get(self,tournamentID,clubID,playerID):  # get stats
        stats = StatsModel.find_stats(tournamentID,clubID,playerID)
        if not stats:
            return {'message' : 'stats not found.'}, 404
        try:
            playerStats = {
                'stats' : stats.json(),
                'gamePerformance' : {
                    "win" : 0,
                    "draw" : 0,
                    "loss" : 0,
                }
            }

            gamePerformance = GamePerformance.get_player_tournament_performance(tournamentID,clubID,playerID)
            if gamePerformance:
                playerStats['gamePerformance'] = gamePerformance
            return playerStats, 200
        except:
            traceback.print_exc()
            return {'message' : 'Internal Server Error. Cannot load player total stats.'}, 500

    def post(self,tournamentID,clubID,playerID): # create a new stats
        stats = StatsModel.find_stats(tournamentID,clubID,playerID)
        if stats:
            return {'message' : 'stats already exists.'}, 400
        stats = StatsModel(tournamentID,clubID,playerID,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0)
        try:
            stats.save_to_db()
            return stats.json(), 201
        except:
            traceback.print_exc()
            return {'message' : 'Internal server error, create stats failed.'}, 500

    def delete(self,tournamentID,clubID,playerID): # delete a stats
        stats = StatsModel.find_stats(tournamentID,clubID,playerID)
        if not stats:
            return {'message' : 'stats not found.'}, 404
        try:
            stats.delete_from_db()
            return {'message' : 'stats deleted.'}, 200
        except:
            traceback.print_exc()
            return {'message' : 'Internal server error, delete stats failed.'}, 500

    def put(self,tournamentID,clubID,playerID):  # update an existing stats
        stats = StatsModel.find_stats(tournamentID,clubID,playerID)
        if not stats:
            return {'message' : 'stats not found'}, 404

        data = self.parser.parse_args()
        vector = StatsModel(tournamentID,clubID,playerID,**data)
        new_stats = StatsModel.get_updated_stats(stats,vector)

        try:
            new_stats.save_to_db()
            return new_stats.json(), 200
        except:
            traceback.print_exc()
            return {'message' : 'Internal server error, update stats failed.'}, 500

class StatsList(Resource):

    def get(self):
        return {'stats':[stats.json() for stats in StatsModel.find_all()]},200

class StatsByPlayer(Resource):

    def get(self,playerID):
        try:
            playerStats = {
                'stats' : StatsModel.find_player_total_stats(playerID),
                'gamePerformance' : {
                    "win" : 0,
                    "draw" : 0,
                    "loss" : 0,
                }
            }
            gamePerformance = GamePerformance.get_player_total_performance(playerID)
            if gamePerformance:
                playerStats['gamePerformance'] = gamePerformance
            return playerStats, 200
        except:
            traceback.print_exc()
            return {'message' : 'Internal Server Error. Cannot load player total stats.'}, 500

class StatsByClubPlayer(Resource):

    def get(self,clubID,playerID):
        try:
            playerStats = {
                'stats' : StatsModel.find_club_player_stats(clubID,playerID),
                'gamePerformance' : {
                    "win" : 0,
                    "draw" : 0,
                    "loss" : 0,
                }
            }
            gamePerformance = GamePerformance.get_player_club_performance(clubID,playerID)
            if gamePerformance:
                playerStats['gamePerformance'] = gamePerformance
            return playerStats, 200
        except:
            traceback.print_exc()
            return {'message' : 'Internal Server Error. Cannot load player total stats.'}, 500

class StatsByTournamentPlayer(Resource):

    def get(self,tournamentID,clubID,playerID):
        try:
            playerStats = {
                'stats' : StatsModel.find_stats(tournamentID,clubID,playerID),
                'gamePerformance' : {
                    "win" : 0,
                    "draw" : 0,
                    "loss" : 0,
                }
            }
            gamePerformance = GamePerformance.get_player_tournament_performance(tournamentID,clubID,playerID)
            if gamePerformance:
                playerStats['gamePerformance'] = gamePerformance
            return playerStats, 200
        except:
            traceback.print_exc()
            return {'message' : 'Internal Server Error. Cannot load player total stats.'}, 500

class StatsByTournamentClub(Resource):

    def get(self,tournamentID,clubID): # get club's total stats in this tournament
        try:
            clubStats = {
                'stats' : StatsModel.find_tournament_club_total_stats(tournamentID,clubID),
                'gamePerformance' : {
                    "win" : 0,
                    "draw" : 0,
                    "loss" : 0,
                    "cleanSheet" : 0,
                    "goalsConceded" : 0,
                }
            }
            gamePerformance = GamePerformance.get_club_tournament_performance(tournamentID,clubID)
            if gamePerformance:
                clubStats['gamePerformance'] = gamePerformance
            return clubStats, 200
        except:
            traceback.print_exc()
            return {'message' : 'Internal Server Error. Cannot load club tournament total stats.'}, 500


"""Methods recording timestamps of newly added data, used for more targeted processing."""
import os
from datetime import datetime, timedelta
import json
from transmission.processing.satellites import TIME_FORMAT

TIME_RANGE_FILES_DIR = "transmission/processing/temp/"

def get_new_data_file_path(satellite: str, link: str) -> str:
    """Return filepath of the new data time range file."""
    return TIME_RANGE_FILES_DIR + satellite + "/" + satellite + "_" + link + ".json"

def get_failed_data_file_path(satellite: str, link: str) -> str:
    """Return filepath of the time range file storing the interval ."""
    return TIME_RANGE_FILES_DIR + satellite + "/" + "failed" + "_" + link + ".json"

def get_new_data_scraper_temp_folder(satellite: str) -> str:
    """Return filepath of the scraper process temp time range files."""
    return TIME_RANGE_FILES_DIR + satellite + "/scraper/"

def get_new_data_buffer_temp_folder(satellite: str) -> str:
    """Return filepath of the buffer process temp time range files."""
    return TIME_RANGE_FILES_DIR + satellite + "/buffer/"

def read_time_range_file(input_file: str) -> dict:
    """Read time range file and return contents as dictionary."""
    new_data_time_range = {}
    with open(input_file, "r", encoding="utf-8") as file:
        new_data_time_range = json.load(file)

    return new_data_time_range

def save_timestamps_to_file(timestamps: dict, input_file: str) -> None:
    """Dump timestamps to the input file in json format."""
    with open(input_file, "w", encoding="utf-8") as file:
        file.write(json.dumps(timestamps, indent=4))

def reset_new_data_timestamps(satellite: str, link: str, input_file: str) -> None:
    """Replace timestamps form the json given by the input file with []."""
    new_data_time_range = read_time_range_file(input_file)
    new_data_time_range[satellite][link] = []

    with open(input_file, "w", encoding="utf-8") as file:
        file.write(json.dumps(new_data_time_range, indent=4))

def include_timestamp_in_time_range(satellite: str, link: str, timestamp,
                                    input_file: str = None, existing_range: dict = None) -> dict:
    """This function ensures that a given timestamp will be included in the
    time range such that it can then be processed and parsed from raw form.
    The range can be maintained in memory given an existing_range or in file given an input_file."""

    if isinstance(timestamp, str):
        time = datetime.strptime(timestamp, TIME_FORMAT)
    else:
        time = timestamp

    start_time = (time - timedelta(seconds=1)).strftime(TIME_FORMAT)
    end_time = (time + timedelta(seconds=1)).strftime(TIME_FORMAT)

    time_range = (start_time, end_time)

    return update_new_data_timestamps(satellite, link, time_range, input_file, existing_range)

def update_new_data_timestamps(satellite: str, link: str, new_time_range: tuple,
                               input_file: str = None, existing_range: dict = None) -> dict:
    """Bookkeep time range of unprocessed telemetry.
     If an input_file is specified, the timestamps from the file, will be updated and dumped.
     If the existing_range is specified, it will be updated and returned as a dictionary.
     If both input_file and existing_range are specified, an exception is raised."""

    if input_file is not None and existing_range is not None:
        raise RuntimeError("Specify either input_file or existing_range, not both.")

    start_time = new_time_range[0]
    end_time = new_time_range[1]

    if input_file is not None:
        new_data_time_range = read_time_range_file(input_file)

    elif existing_range in [{}, None]:
        new_data_time_range = {}
        new_data_time_range[satellite] = {}
        new_data_time_range[satellite][link] = []
    else:
        new_data_time_range = existing_range

    # No time range saved
    if new_data_time_range[satellite][link] == []:
        new_data_time_range[satellite][link] = [start_time, end_time]
    # Update time range
    else:
        new_data_time_range[satellite][link][0] = min(
            new_data_time_range[satellite][link][0],
            start_time
        )
        new_data_time_range[satellite][link][1] = max(
            new_data_time_range[satellite][link][1],
            end_time
        )
    if input_file is not None:
        with open(input_file, "w", encoding="utf-8") as file:
            json.dump(new_data_time_range, file, indent=4)

    return new_data_time_range

def combine_time_ranges(satellite: str, link: str) -> None:
    """Combine time ranges of new data from all processes (buffer processing and scraper)."""
    scraper_folder = get_new_data_scraper_temp_folder(satellite)
    buffer_folder = get_new_data_buffer_temp_folder(satellite)

    for folder in [scraper_folder, buffer_folder]:

        for temp_file in os.listdir(folder):
            if link in temp_file:
                new_data_time_range = read_time_range_file(folder + temp_file)
                new_data_overview_file = get_new_data_file_path(satellite, link)
                include_timestamp_in_time_range(satellite, link,
                                                new_data_time_range[satellite][link][0],
                                                input_file=new_data_overview_file)
                include_timestamp_in_time_range(satellite, link,
                                                new_data_time_range[satellite][link][1],
                                                input_file=new_data_overview_file)
                os.remove(folder + temp_file)


import os

import paddle.fluid as fluid
from paddle.fluid.dygraph import Conv2D, Pool2D
from paddle.nn import SyncBatchNorm as BatchNorm

from dygraph.cvlibs import manager
from dygraph import utils

class UNet(fluid.dygraph.Layer):
    """
    U-Net: Convolutional Networks for Biomedical Image Segmentation.
    https://arxiv.org/abs/1505.04597

    Args:
        num_classes (int): the unique number of target classes.
        pretrained_model (str): the path of pretrained model.
        ignore_index (int): the value of ground-truth mask would be ignored while computing loss or doing evaluation. Default 255.
    """

    def __init__(self, num_classes, pretrained_model=None, ignore_index=255):
        super(UNet, self).__init__()
        self.encode = UnetEncoder()
        self.decode = UnetDecode()
        self.get_logit = GetLogit(64, num_classes)
        self.ignore_index = ignore_index
        self.EPS = 1e-5

        self.init_weight(pretrained_model)

    def forward(self, x, label=None):
        encode_data, short_cuts = self.encode(x)
        decode_data = self.decode(encode_data, short_cuts)
        logit = self.get_logit(decode_data)
        if self.training:
            return self._get_loss(logit, label)
        else:
            score_map = fluid.layers.softmax(logit, axis=1)
            score_map = fluid.layers.transpose(score_map, [0, 2, 3, 1])
            pred = fluid.layers.argmax(score_map, axis=3)
            pred = fluid.layers.unsqueeze(pred, axes=[3])
            return pred, score_map

    def init_weight(self, pretrained_model=None):
        """
        Initialize the parameters of model parts.
        Args:
            pretrained_model ([str], optional): the pretrained_model path of backbone. Defaults to None.
        """
        if pretrained_model is not None:
            if os.path.exists(pretrained_model):
                utils.load_pretrained_model(self.backbone, pretrained_model)
                utils.load_pretrained_model(self, pretrained_model)
            else:
                raise Exception('Pretrained model is not found: {}'.format(
                    pretrained_model))

    def _get_loss(self, logit, label):
        logit = fluid.layers.transpose(logit, [0, 2, 3, 1])
        label = fluid.layers.transpose(label, [0, 2, 3, 1])
        mask = label != self.ignore_index
        mask = fluid.layers.cast(mask, 'float32')
        loss, probs = fluid.layers.softmax_with_cross_entropy(
            logit,
            label,
            ignore_index=self.ignore_index,
            return_softmax=True,
            axis=-1)

        loss = loss * mask
        avg_loss = fluid.layers.mean(loss) / (
            fluid.layers.mean(mask) + self.EPS)

        label.stop_gradient = True
        mask.stop_gradient = True
        return avg_loss

class UnetEncoder(fluid.dygraph.Layer):
    def __init__(self):
        super(UnetEncoder, self).__init__()
        self.double_conv = DoubleConv(3, 64)
        self.down1 = Down(64, 128)
        self.down2 = Down(128, 256)
        self.down3 = Down(256, 512)
        self.down4 = Down(512, 512)

    def forward(self, x):
        short_cuts = []
        x = self.double_conv(x)
        short_cuts.append(x)
        x = self.down1(x)
        short_cuts.append(x)
        x = self.down2(x)
        short_cuts.append(x)
        x = self.down3(x)
        short_cuts.append(x)
        x = self.down4(x)
        return x, short_cuts

class UnetDecode(fluid.dygraph.Layer):
    def __init__(self):
        super(UnetDecode, self).__init__()
        self.up1 = Up(512, 256)
        self.up2 = Up(256, 128)
        self.up3 = Up(128, 64)
        self.up4 = Up(64, 64)

    def forward(self, x, short_cuts):
        x = self.up1(x, short_cuts[3])
        x = self.up2(x, short_cuts[2])
        x = self.up3(x, short_cuts[1])
        x = self.up4(x, short_cuts[0])
        return x

class DoubleConv(fluid.dygraph.Layer):
    def __init__(self, num_channels, num_filters):
        super(DoubleConv, self).__init__()
        self.conv0 = Conv2D(
            num_channels=num_channels,
            num_filters=num_filters,
            filter_size=3,
            stride=1,
            padding=1)
        self.bn0 = BatchNorm(num_filters)
        self.conv1 = Conv2D(
            num_channels=num_filters,
            num_filters=num_filters,
            filter_size=3,
            stride=1,
            padding=1)
        self.bn1 = BatchNorm(num_filters)

    def forward(self, x):
        x = self.conv0(x)
        x = self.bn0(x)
        x = fluid.layers.relu(x)
        x = self.conv1(x)
        x = self.bn1(x)
        x = fluid.layers.relu(x)
        return x

class Down(fluid.dygraph.Layer):
    def __init__(self, num_channels, num_filters):
        super(Down, self).__init__()
        self.max_pool = Pool2D(
            pool_size=2, pool_type='max', pool_stride=2, pool_padding=0)
        self.double_conv = DoubleConv(num_channels, num_filters)

    def forward(self, x):
        x = self.max_pool(x)
        x = self.double_conv(x)
        return x

class Up(fluid.dygraph.Layer):
    def __init__(self, num_channels, num_filters):
        super(Up, self).__init__()
        self.double_conv = DoubleConv(2 * num_channels, num_filters)

    def forward(self, x, short_cut):
        short_cut_shape = fluid.layers.shape(short_cut)
        x = fluid.layers.resize_bilinear(x, short_cut_shape[2:])
        x = fluid.layers.concat([x, short_cut], axis=1)
        x = self.double_conv(x)
        return x

class GetLogit(fluid.dygraph.Layer):
    def __init__(self, num_channels, num_classes):
        super(GetLogit, self).__init__()
        self.conv = Conv2D(
            num_channels=num_channels,
            num_filters=num_classes,
            filter_size=3,
            stride=1,
            padding=1)

    def forward(self, x):
        x = self.conv(x)
        return x

@manager.MODELS.add_component
def unet(*args, **kwargs):
    return UNet(*args, **kwargs)