import numpy
import random
import torch
import torch.nn as nn
import torch.nn.functional as F
import pandas
from torch_geometric.data import DataLoader
from sklearn.preprocessing import scale
from sklearn.metrics import mean_absolute_error
from sklearn.metrics import mean_squared_error
from scipy.spatial.distance import correlation
from torch.utils.data import TensorDataset
from torch.utils.data import DataLoader
from ane.util.data import normalize
from ane.geomagnetic_strom_forecasting.storm_detection import detect_storm

class FCNN(nn.Module):
    def __init__(self, dim_in, dim_out):
        super(FCNN, self).__init__()
        self.fc1 = nn.Linear(dim_in, 128)
        self.bn1 = nn.BatchNorm1d(128)
        self.fc2 = nn.Linear(128, 128)
        self.bn2 = nn.BatchNorm1d(128)
        self.fc3 = nn.Linear(128, dim_out)

    def forward(self, x):
        h = F.relu(self.bn1(self.fc1(x)))
        h = F.relu(self.bn2(self.fc2(h)))
        out = self.fc3(h)

        return out

    def fit(self, data_loader, optimizer, criterion):
        self.train()
        sum_train_losses = 0

        for data, targets in data_loader:
            data = data.cuda()
            targets = targets.cuda()

            preds = self(data)
            loss = criterion(preds, targets)

            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

            sum_train_losses += loss.item()

        return sum_train_losses / len(data_loader)

    def predict(self, data_loader):
        self.eval()
        list_preds = list()

        with torch.no_grad():
            for data, _ in data_loader:
                data = data.cuda()

                preds = self(data)
                list_preds.append(preds)

        return torch.cat(list_preds, dim=0)

class RNNModel(nn.Module):
    def __init__(self, dim_in, dim_hidden, n_layers, len_seq, dim_emb, rnn_model):
        super(RNNModel, self).__init__()
        self.dim_hidden = dim_hidden
        self.n_layers = n_layers
        self.rnn_model = rnn_model

        if self.rnn_model == 'rnn':
            self.rnn = nn.RNN(dim_in, dim_hidden, n_layers, batch_first=True)
        elif self.rnn_model == 'lstm':
            self.rnn = nn.LSTM(dim_in, dim_hidden, n_layers, batch_first=True)
        elif self.rnn_model == 'gru':
            self.rnn = nn.GRU(dim_in, dim_hidden, n_layers, batch_first=True)

        self.bn1 = nn.BatchNorm1d(dim_hidden * len_seq)
        self.fc1 = nn.Linear(dim_hidden * len_seq, dim_emb)

    def forward(self, x):
        h_0 = torch.zeros(self.n_layers, x.shape[0], self.dim_hidden).cuda()

        if self.rnn_model == 'lstm':
            c_0 = torch.zeros(self.n_layers, x.shape[0], self.dim_hidden).cuda()
            z, _ = self.rnn(x, (h_0, c_0))
        else:
            z, _ = self.rnn(x, h_0)

        z = self.bn1(z.reshape(z.shape[0], -1))
        embs = self.fc1(z)

        return embs

class Transformer(nn.Module):
    def __init__(self, dim_in, dim_emb, n_layers=None, len_seq=None):
        super(Transformer, self).__init__()
        self.rnn_model = 'transformer'
        self.n_layers = n_layers
        self.rnn = nn.Transformer(dim_in, 7, 2, 2, 2, 0.05, batch_first=True)
        self.bn1 = nn.BatchNorm1d(dim_in * len_seq)
        self.fc1 = nn.Linear(dim_in * len_seq, dim_emb)

    def forward(self, src, tgt):
        z = self.rnn(src, tgt)
        z = self.bn1(z.reshape(z.shape[0], -1))
        embs = self.fc1(z)

        return embs

def get_seq_dataset(dataset, len_seq):
    seq_dataset = list()
    seq = list()

    for i in range(0, len_seq):
        seq.append(numpy.zeros(dataset.shape[1]))

    for i in range(0, dataset.shape[0]):
        seq.pop(0)
        seq.append(dataset[i, :])
        seq_dataset.append(numpy.hstack(seq))

    return numpy.vstack(seq_dataset)

def get_data_loader(*data, batch_size, shuffle=False, dtype=torch.float):
    tensors = [torch.tensor(d, dtype=dtype) for d in data]
    return DataLoader(TensorDataset(*tuple(tensors)), batch_size=batch_size, shuffle=shuffle)

def get_pos_neg_samples(seq, targets, tgt=None):
    pos_seq = list()
    pos_tgt = list()
    pos_targets = list()
    neg_seq = list()
    neg_tgt = list()
    neg_targets = list()

    for i in range(0, seq.shape[0]):
        idx = random.sample(range(0, seq.shape[0]), 2)

        if torch.norm(targets[i] - targets[idx[0]]) < torch.norm(targets[i] - targets[idx[1]]):
            pos_seq.append(seq[idx[0], :, :].view(1, seq.shape[1], seq.shape[2]))
            pos_targets.append(targets[idx[0]])
            neg_seq.append(seq[idx[1], :, :].view(1, seq.shape[1], seq.shape[2]))
            neg_targets.append(targets[idx[1]])

            if tgt is not None:
                pos_tgt.append(tgt[idx[0], :, :].view(1, tgt.shape[1], tgt.shape[2]))
                neg_tgt.append(tgt[idx[1], :, :].view(1, tgt.shape[1], tgt.shape[2]))
        else:
            pos_seq.append(seq[idx[1], :, :].view(1, seq.shape[1], seq.shape[2]))
            pos_targets.append(targets[idx[1]])
            neg_seq.append(seq[idx[0], :, :].view(1, seq.shape[1], seq.shape[2]))
            neg_targets.append(targets[idx[0]])

            if tgt is not None:
                pos_tgt.append(tgt[idx[1], :, :].view(1, tgt.shape[1], tgt.shape[2]))
                neg_tgt.append(tgt[idx[0], :, :].view(1, tgt.shape[1], tgt.shape[2]))
    if tgt is None:
        return torch.vstack(pos_seq), torch.vstack(pos_targets), torch.vstack(neg_seq), torch.vstack(neg_targets)
    else:
        return torch.vstack(pos_seq), torch.vstack(pos_tgt), torch.vstack(pos_targets),\
               torch.vstack(neg_seq), torch.vstack(neg_tgt), torch.vstack(neg_targets)

def loss_ane(seq, targets, emb_net, tgt=None):
    rand_idx = torch.randperm(seq.shape[0])
    seq_sample = seq[rand_idx, :, :]
    targets_sample = targets[rand_idx]

    if tgt is None:
        emb_data = F.normalize(emb_net(seq), p=2, dim=1)
        emb_sample = F.normalize(emb_net(seq_sample), p=2, dim=1)
    else:
        tgt_sample = tgt[rand_idx, :, :]
        emb_data = F.normalize(emb_net(seq, tgt), p=2, dim=1)
        emb_sample = F.normalize(emb_net(seq_sample, tgt_sample), p=2, dim=1)

    dist_emb = torch.sum((emb_data - emb_sample)**2, dim=1)
    dist_target = torch.sum((targets - targets_sample)**2, dim=1)

    return torch.mean((dist_emb - dist_target)**2)

def train(emb_net, data_loader, optimizer):
    if emb_net.rnn_model == 'transformer':
        return __train_transformer(emb_net, data_loader, optimizer)
    else:
        return __train_rnn(emb_net, data_loader, optimizer)

def __train_rnn(emb_net, data_loader, optimizer):
    emb_net.train()
    train_loss = 0

    for seq, targets in data_loader:
        seq = seq.cuda()
        targets = targets.cuda()
        loss = loss_ane(seq, targets, emb_net)

        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        train_loss += loss.item()

    return train_loss / len(data_loader)

def __train_transformer(emb_net, data_loader, optimizer):
    emb_net.train()
    train_loss = 0

    for batch in data_loader:
        src = batch[0].cuda()
        tgt = batch[1].cuda()
        targets = batch[2].cuda()
        loss = loss_ane(src, targets, emb_net, tgt=tgt)

        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        train_loss += loss.item()

    return train_loss / len(data_loader)

def test(emb_net, data_loader):
    if emb_net.rnn_model == 'transformer':
        return __test_transformer(emb_net, data_loader)
    else:
        return __test_rnn(emb_net, data_loader)

def __test_rnn(emb_net, data_loader):
    emb_net.eval()
    list_embs = list()

    with torch.no_grad():
        for seq, _ in data_loader:
            seq = seq.cuda()

            embs = F.normalize(emb_net(seq), p=2, dim=1)
            list_embs.append(embs)

    return torch.cat(list_embs, dim=0)

def __test_transformer(emb_net, data_loader):
    emb_net.eval()
    list_embs = list()

    with torch.no_grad():
        for batch in data_loader:
            src = batch[0].cuda()
            tgt = batch[1].cuda()

            embs = F.normalize(emb_net(src, tgt), p=2, dim=1)
            list_embs.append(embs)

    return torch.cat(list_embs, dim=0)

def get_seq_data(x, y, len_seq):
    x_seq = []
    y_seq = []

    for i in range(0, x.shape[0] - len_seq):
        x_seq.append(x[i:i+len_seq, :])
        y_seq.append(y[i+len_seq])

    return torch.FloatTensor(x_seq), torch.FloatTensor(y_seq).view([-1, 1])

def exec_dml(dataset_name, rnn_model, init_lr_emb, l2_emb, bs_emb, dim_emb,
             init_lr_pred, l2_pred, bs_pred):
    # Experiment settings.
    path_save_file = 'results/' + dataset_name
    len_seq = 8

    # Load dataset.
    data = numpy.array(pandas.read_excel('datasets/' + dataset_name + '.xlsx'))
    x = scale(data[:, 1:-1])
    y = data[:, -1]
    y_norm = normalize(y)
    n_train = int(0.8 * x.shape[0])

    if rnn_model == 'transformer':
        x_seq, y_pred = get_seq_data(x, y_norm, len_seq=len_seq)
        y_seq = torch.vstack([x_seq[1:, :], torch.zeros((1, len_seq, x.shape[1]))])
        loader_train = DataLoader(TensorDataset(x_seq[:n_train], y_seq[:n_train], y_pred[:n_train]), batch_size=bs_emb)
        loader_test = DataLoader(TensorDataset(x_seq[n_train:], y_seq[n_train:], y_pred[n_train:]), batch_size=bs_emb)
        emb_net = Transformer(dim_in=x.shape[1], dim_emb=dim_emb, len_seq=len_seq).cuda()
    else:
        x_seq, y_seq = get_seq_data(x, y_norm, len_seq=len_seq)
        loader_train = DataLoader(TensorDataset(x_seq[:n_train], y_seq[:n_train]), batch_size=bs_emb, shuffle=False)
        loader_test = DataLoader(TensorDataset(x_seq[n_train:], y_seq[n_train:]), batch_size=bs_emb, shuffle=False)
        emb_net = RNNModel(rnn_model=rnn_model, dim_emb=dim_emb, dim_in=x.shape[1],
                           dim_hidden=16, n_layers=2, len_seq=len_seq).cuda()

    optimizer = torch.optim.Adam(emb_net.parameters(), lr=init_lr_emb, weight_decay=l2_emb)

    # Train the embedding network.
    for epoch in range(0, 500):
        train_loss = train(emb_net, loader_train, optimizer)
        print('Epoch [{}/{}]\tTrain loss: {:.4f}'.format(epoch + 1, 500, train_loss))

    # Generate the latent embeddings of the data using the trained embedding network.
    emb_train = test(emb_net, loader_train).cpu().numpy()
    emb_test = test(emb_net, loader_test).cpu().numpy()

    # Save embedding results.
    df_train = pandas.DataFrame(numpy.hstack([emb_train, y[:n_train].reshape(-1, 1)]))
    df_test = pandas.DataFrame(numpy.hstack([emb_test, y[n_train:-len_seq].reshape(-1, 1)]))
    df_train.to_excel(path_save_file + '/embs_train.xlsx', header=None, index=None)
    df_test.to_excel(path_save_file + '/embs_test.xlsx', header=None, index=None)

    # Load embedding results.
    dataset_train = numpy.array(pandas.read_excel(path_save_file + '/embs_train.xlsx', header=None))
    dataset_test = numpy.array(pandas.read_excel(path_save_file + '/embs_test.xlsx', header=None))
    dataset_x = scale(numpy.vstack([dataset_train[:, :-1], dataset_test[:, :-1]]))
    dataset_train_x = dataset_x[:dataset_train.shape[0], :]
    dataset_test_x = dataset_x[dataset_train.shape[0]:, :]
    dataset_train_x_seq = get_seq_dataset(dataset_train_x, len_seq)
    dataset_test_x_seq = get_seq_dataset(dataset_test_x, len_seq)
    dataset_train_y = dataset_train[:, -1].reshape(-1, 1)
    dataset_test_y = dataset_test[:, -1].reshape(-1, 1)

    # Load the dataset on the embedding space.
    data_loader_train = get_data_loader(dataset_train_x_seq, dataset_train_y, batch_size=bs_pred, shuffle=True)
    data_loader_test = get_data_loader(dataset_test_x_seq, dataset_test_y, batch_size=bs_pred)

    # Define prediction model and optimizer.
    model = FCNN(dataset_train_x_seq.shape[1], dim_out=1).cuda()
    optimizer = torch.optim.Adam(model.parameters(), lr=init_lr_pred, weight_decay=l2_pred)
    criterion = torch.nn.MSELoss()

    # Train the prediction model.
    for epoch in range(0, 500):
        train_loss = model.fit(data_loader_train, optimizer, criterion)
        print(epoch, train_loss)

    # Evaluate the trained model.
    preds_test = model.predict(data_loader_test).cpu().numpy()
    mae = mean_absolute_error(dataset_test_y, preds_test)
    rmse = numpy.sqrt(mean_squared_error(dataset_test_y, preds_test))
    corr = correlation(dataset_test_y, preds_test)

    # Save the trained model and the extrapolation results.
    df = pandas.DataFrame(numpy.hstack([dataset_test_y, preds_test]))
    df.to_excel(path_save_file + '/preds.xlsx', index=None, header=['targets', 'preds'])
    precision, recall, f1 = detect_storm(path_save_file)

    return mae, rmse, corr, precision, recall, f1


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('autumnbreeze', '0001_initial'),
    ]

    operations = [
        migrations.AddField(
            model_name='comparingoption',
            name='location_name',
            field=models.CharField(default='test', max_length=100),
            preserve_default=False,
        ),
        migrations.AlterField(
            model_name='comparingoption',
            name='fluctuation',
            field=models.PositiveSmallIntegerField(help_text='An adjustable number of fluctuation in percentage to calculate         fluctuation between two data sets. F. ex. if number 20 will be saved         in this field, when two files will be compared, the system will try         to track a 20% increase of location_ids that were mentioned in a         particular day.'),
        ),
    ]


import json

from django.http import HttpResponse
from django.core import serializers
from SeusPedidos.App.models.cliente import Cliente as ClienteModel
from SeusPedidos.App.models.pedido import Pedido as PedidoModel
from SeusPedidos.App.core.apiview import ApiView
from SeusPedidos.App.form.cliente import ClienteForm

class Cliente(ApiView):

    def get(self, request):
        id = request.GET.get('id')
        try:
            if not id:
                result = ClienteModel.objects.all()
            else:
                result = ClienteModel.objects.get(id=id)
        except ClienteModel.DoesNotExist:
            result = {}

        return HttpResponse(
            serializers.serialize('json', result), mimetype='application/json'
        )

    def post(self, request):
        form = ClienteForm(self._postData)
        if form.is_valid():
            id = self._postData.get('id')
            if (id == None):
                try:
                    cliente = ClienteModel.objects.create(
                        nome=self._postData.get('nome'),
                        email=self._postData.get('email')
                    )
                    if (cliente != None):
                        result = self._apiresult.success(None)
                    else:
                        result = self._apiresult.error(None)
                except Exception:
                    result = self._apiresult.error(None)
            else:
                try:
                    cliente = ClienteModel.objects.get(id=id)
                    cliente.nome = self._postData.get('nome', cliente.nome)
                    cliente.email = self._postData.get('email', cliente.email)
                    cliente.save()
                    result = self._apiresult.success(None)
                except Exception:
                    result = self._apiresult.error(None)
        else:
            result = self._apiresult.error(
                form.errors
            )

        return HttpResponse(
            json.dumps(result), mimetype='application/json'
        )

    def delete(self, request):
        id = request.GET.get('id')
        total = PedidoModel.objects.filter(cliente=id).count()
        if (total == 0):
            try:
                cliente = ClienteModel.objects.get(id=id)
                cliente.delete()
                result = self._apiresult.success(None)
            except:
                result = self._apiresult.error(None)
        else:
            result = self._apiresult.error(None)
        return HttpResponse(
            json.dumps(result), mimetype='application/json'
        )


#!/usr/bin/env python3

import socket
import threading
import csv
import json
import argparse
import sys
import time
import datetime

class ThreadedServer(object):
    def __init__(self, host, opt):
        self.device = {}
        self.device['NoMode'] = {'points': 0}
        self.device['label'] = {'label': 0, 'points': 0}
        self.host = host
        self.port = opt.port
        self.opt = opt
        self.state = self.device[opt.mode if opt.mode else 'NoMode']
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        self.sock.bind((self.host, self.port))
        self.lock = threading.Lock()
        self.check = -1

    def listen(self):
        self.sock.listen(5) # server prepares a queue of 5
        while True:
            client, address = self.sock.accept()
            print(f'Connection from {address} has been established! ')
            client.settimeout(500)
            threading.Thread(target=self.listenToClient, args=(client, address)).start()
            threading.Thread(target=self.sendStreamToClient, args=
            (client, self.sendCSVfile())).start()

    def handle_client_answer(self, obj):
        if self.opt.mode is not None and self.opt.mode == 'label':
            if 'label' not in obj:
                return
            self.lock.acquire()
            if self.state['label'] == int(obj['label']):
                self.state['points'] += 1
                self.check = 1
            else:
                self.check = 0
            self.lock.release()
        return

    def listenToClient(self, client, address):
        size = 1024
        total = 0
        while True:
            try:
                data = client.recv(size).decode()

                if data:
                    a = json.loads(data.rstrip('\n\r '))
                    self.handle_client_answer(a)
                    total += 1
                    print(f"Correctly predicted: {self.state['points']} records.")
                    print(f"Accuracy rate: {self.state['points']/total*100}%")
                else:
                    print('Client disconnected')
                    return False

            except:
                print(f"so far we have {self.state['points']} points")
                print('Client closed the connection')
                print("Unexpected error:", sys.exc_info()[0])
                client.close()
                return False

    def handleCustomData(self, buffer):
        if self.opt.mode is not None and self.opt.mode == 'label':
            self.lock.acquire()
            self.state['label'] = int(buffer['label'])
            buffer['label'] = self.check
            self.lock.release()

    def sendStreamToClient(self, client, buffer):
        for i in buffer:
            print(i)
            self.handleCustomData(i)
            try:
                client.send((self.convertStringToJSON(i) + '\n').encode('utf-8'))
                time.sleep(self.opt.interval)
            except:
                print('End of stream')
                return False
        client.send((self.convertStringToJSON(self.state) + '\n').encode('utf-8'))
        return False

    def convertStringToJSON(self, st):
        return json.dumps(st)

    def sendCSVfile(self):
        out = []
        for f in self.opt.files:
            print('reading file %s...' % f)
            csvfile = open(f, 'r')
            reader = csv.DictReader(csvfile)
            for row in reader:
                out += [row]
        return out

if __name__ == "__main__":

    parser = argparse.ArgumentParser(usage='python tcp_server_project.py -p 9998 -f testing.csv -t 1 -m label')
    #parser = argparse.ArgumentParser(usage='usage: tcp_server -p port [-f -m]')
    #parser = argparse.ArgumentParser(usage='usage: tcp_server.py -p 9998 -f occupancy/censortraining.csv -t 1')
    parser.add_argument('-f', '--files', nargs='+')
    parser.add_argument("-m", "--mode", action="store", dest="mode")
    parser.add_argument("-p", "--port", action="store", dest="port", type=int)
    parser.add_argument("-t", "--time-interval", action="store",
                        dest="interval", type=int, default=1)

    opt = parser.parse_args()
    #host = opt.host
    #port = opt.port
    if not opt.port:
        parser.error('Port not given')
    ThreadedServer('localhost', opt).listen()

'''
{"wrist": "0", "acceleration_x": "0.265", "acceleration_y": "-0.7814", "acceleration_z": "-0.0076",
"gyro_x": "-0.059", "gyro_y": "0.0325", "gyro_z": "-2.9296", "label": "0"}

{"CO2": "760.4", "Temperature": "23.718", "Light": "578.4", "Number": "141",
"Occupancy": "1", "Humidity": "26.29", "HumidityRatio": "0.00477266099212519",
"date": "2015-02-02 14:19:59"}

'''


"""
Scraping Engine creates the Identiy Data for the papers.
The Mining Engine on Instantiation
    - Will check For the New Papers to Mine.
    - It will Create a ArxivPaper
                --> Download Latex
                --> Parse Latex
                --> Sematically Parse the Paper here too.
                --> Mining Ontology Here Too
"""
from .database import ArxivDatabase
from .record import ArxivRecord,ArxivIdentity,ArxivSematicParsedResearch,Ontology,Author
from .logger import create_logger
from .exception import ArxivAPIException
from .ontology_miner import OntologyMiner
from .paper import ArxivPaper,ResearchPaperFactory
import time
from multiprocessing import Process,Event
from signal import signal, SIGINT
import random
import string
from typing import List

def random_string(stringLength=8):
    letters = string.ascii_lowercase
    return ''.join(random.choice(letters) for i in range(stringLength))

class MiningEngine:
    """
    Run as Isolated Process
        - Query Database For Unmined Paper
        - Mine Paper : With ArxivPaper Object.
            - If Arxiv Acts Bitchy with 500 Errors Wait and Mine.
    """
    def __init__(self,\
            database:ArxivDatabase,\
            data_root_path,\
            detex_path):

        self.db = database
        self.data_root_path = data_root_path
        self.detex_path = detex_path
        self.logger = create_logger(self.__class__.__name__+"__"+random_string())

    def mine_record(self,paper_record:ArxivRecord):
        paper_obj = ArxivPaper.from_arxiv_record(self.data_root_path,\
                                                    paper_record,\
                                                    detex_path=self.detex_path)
        try:
            paper_obj.mine_paper()
        except Exception as e:
            self.logger.error('Failed Mining Paper : %s\n\n%s'%(paper_obj.identity.identity,str(e)))
            return None

        return paper_obj

    def _paper_mining_logic(self):
        """_paper_mining_logic
        1. Get unmined ArxivRecord
        2. Create `AxivPaper` from `ArxivRecord` and mine it
        3. Create `ArxivSematicParsedResearch` from `ArxivRecord` and save it .
        3. save Record and Mark as Mined.
        """
        paper_record = self.db.get_unmined_paper()
        paper_mined = False
        if not paper_record:
            return None,paper_mined

        self.logger.info("Mining Paper %s"%paper_record.identity.identity)
        paper_obj = self.mine_record(paper_record)

        if paper_obj is not None:
            paper_mined = True
            self.db.save_record(paper_obj.to_arxiv_record())
            paper_record = paper_obj.to_arxiv_record()

        ontology = Ontology()
        if OntologyMiner.is_minable:
            ontology = OntologyMiner.mine_paper(paper_record.identity)
            try:
                self.db.set_many_ontology(ontology.union)
            except:
                self.logger.info("No Ontology Saved")
        else:
            self.logger.info("No Ontology Saved")

        try:
            self.db.set_many_authors( [Author(name=xp) for xp in paper_record.identity.authors])
        except:
            self.logger.info("No Author Saved")
        self.db.set_semantic_parsed_research(ArxivSematicParsedResearch(\
            identity=paper_record.identity,\
            research_object=ResearchPaperFactory.from_arxiv_record(paper_record),\
            ontology=ontology
        ))
        self.db.set_mined(paper_record.identity,paper_mined)

        return paper_record,paper_mined

class SourceHarvestingEngine:
    """
    Run as Isolated Process
        - Given a List of Ids, It will download the Tar source and Put it into a folder. Otherwise it will wait for arxiv to Forgive :)

    Args :
        id_list : List[str] : list of strings that will be used for the
    """
    def __init__(self,id_list:List[str],data_root_path,error_sleep_time=5,scrape_sleep_time=3):
        super().__init__()
        self.id_list = id_list
        self.data_root_path = data_root_path
        self.logger = create_logger(self.__class__.__name__+"__"+random_string())
        self.error_sleep_time = error_sleep_time
        self.scrape_sleep_time = scrape_sleep_time

    def harvest_one(self,arxiv_id:str):
        paper = ArxivPaper(arxiv_id,self.data_root_path,build_paper=False)
        download_path = paper.download_latex()
        return download_path

    def harvest(self):
        harvest_papers_paths = []
        retry_map = {

        }
        while len(self.id_list) > 0:
            arxiv_id = self.id_list.pop()
            try:
                download_path = self.harvest_one(arxiv_id)
                harvest_papers_paths.append((arxiv_id,download_path))
                time.sleep(self.scrape_sleep_time)
            except Exception as e: # Upon exception. Try 3 times by adding it back to list. If still Failure then dont use it.
                self.logger.error(f"Latex Download {str(e)} For ID {arxiv_id}. Will be Sleeping for {self.error_sleep_time}")
                if arxiv_id in retry_map:
                    if retry_map[arxiv_id] > 3:
                        continue
                    else:
                        retry_map[arxiv_id]+=1
                        self.id_list.append(arxiv_id)
                else:
                    retry_map[arxiv_id]=1
                    self.id_list.append(arxiv_id)
                time.sleep(self.error_sleep_time)

        return harvest_papers_paths

class MiningProcess(Process,MiningEngine):
    def __init__(self,
            database:ArxivDatabase,\
            data_root_path,\
            detex_path,\
            mining_interval=5,\
            mining_limit=30,
            empty_wait_time = 600,
            sleep_interval_count = 10):

        # Instantiate The Processes
        Process.__init__(self,daemon=False) # Making it a deamon process.
        MiningEngine.__init__(self,database,data_root_path,detex_path)

        self.mining_interval = mining_interval
        self.empty_wait_time = empty_wait_time
        self.mining_limit = mining_limit
        self.num_mined = 0
        self.exit = Event()
        self.sleep_interval_count = sleep_interval_count
        signal(SIGINT, self.shutdown)

    def run(self):
        """run
        This will Run the MongoEngine's Miner
        """
        self.start_mining()

    def shutdown(self,signal_received, frame):
        # Handle any cleanup here
        self.logger.info('SIGINT or CTRL-C detected. Exiting gracefully')
        self.exit.set()
        exit(0)

    def start_mining(self):
        while True:
            if self.mining_limit is not None:
                if self.num_mined == self.mining_limit:
                    break
            if self.exit.is_set():
                break
            # Sleep Every `sleep_interval_count` records
            if self.num_mined % self.sleep_interval_count == 0 and self.num_mined > 0:
                time.sleep(self.empty_wait_time)
            time.sleep(self.mining_interval)
            paper_record,mined_status = self._paper_mining_logic()
            self.num_mined+=1
            if not paper_record: # Sleep If DB says There are Unmined Papers
                self.logger.info("No Record Found Sleeping For %d"%self.empty_wait_time)
                time.sleep(self.empty_wait_time)
                continue

            if mined_status is False:
                self.logger.error('Couldnt Mine Paper  : %s'%paper_record.identity.identity)
                time.sleep(self.empty_wait_time)
                continue
            self.logger.info('Saved Paper To DB : %s Completed Mining %d Paper'%(paper_record.identity.identity,self.num_mined))

        self.logger.info('Miner Mined : %d'%self.num_mined)


from django.urls import include, re_path, path
from procentric_connect.views import *

urlpatterns = [

    #path('', views.index, name='index'),

    # Url pattern for Pro:Centric TwoWay PMS API
    re_path(r'^pms/v2/check', PMS2WayCheck.as_view(), name='check'),
    re_path(r'^pms/v2/details',PMS2WayDetails.as_view(), name='details'),
    #re_path(r'^pms/v2/host',PMS2WayHost.as_view(), name=host),
    #re_path(r'^pms/v2/site',PMS2WaySite.as_view(), name='site'),
    re_path(r'^pms/v2/statuses', PMS2WayStatus.as_view(), name='statuses'),
    path(r'pms/v2/rooms/<room_id>',PMS2WayRoom.as_view(), name='details'),
    path(r'pms/v2/rooms/<room_id>/folios/<guest_id>',PMS2WayFolios.as_view(), name='folios'),

    re_path(r'^pms/v2/subscriptions',PMS2WaySubscriptions.as_view(), name='subscriptions'),

    #re_path(r'^pms/v2/init',PMS2WayInit.as_view(), name='init),

    # Url pattern for Pro:Centric Webhooks.
    re_path(r'^pms/v2/webhooks/checkin', PMS2WayWebhooks.as_view(), name='checkin'),
    re_path(r'^pms/v2/webhooks/checkout', PMS2WayWebhooks.as_view(), name='checkout'),
    re_path(r'^pms/v2/webhooks/moveroom', PMS2WayWebhooks.as_view(), name='moveroom'),
    re_path(r'^pms/v2/webhooks/getrooms', PMS2WayWebhooks.as_view(), name='getrooms'),
    re_path(r'^pms/v2/webhooks/update', PMS2WayWebhooks.as_view(), name='update'),
]