"""
Módulo para la creación de los resultados en forma gráfica
especificamente en forma de tabla para la automatización
de los ERDs
"""
from tkinter import *
import customtkinter as ctk

def TableResults(data_list) -> None:
    """
    Crea una interfaz en forma de tabla con los resultados obtenidos

    Args:
        data_list : Lista que contiene la lista de los datos obtenidos
        ticketName: Nombre del titulo que aparecerá en la pantalla de la UI

    Return:
        None
    """
    ctk.set_appearance_mode("dark")
    ctk.set_default_color_theme("dark-blue")

    total_rows = len(data_list)
    total_columns = len(data_list[0])

    class Table:
        def __init__(self, root):
            for i in range(total_rows):
                for j in range(total_columns):
                    if j in [0, 1, 7]:
                        widthColumns = 115
                    elif j == 3:
                        widthColumns = 100
                    elif j == 4 or j == 2:
                        widthColumns = 230
                    else:
                        widthColumns = 210

                    if data_list[i][j] in ["Date",'Time','Action',"ERD", "Expected Data", "Data", "Data to Write", "Result", "Comments"]:
                        self.Entry = ctk.CTkEntry(root, width=widthColumns, height=23, corner_radius=2,
                                    font=('Lato',13, "bold"), bg_color="black", fg_color="#1E1E1E",
                                    border_color="#3E3E3E", border_width=1, text_color="white", justify="center")
                    elif data_list[i][j] == "PASS":
                        self.Entry = ctk.CTkEntry(root, width=widthColumns, height=23, corner_radius=2,
                                    font=('Lato',13,'bold'), bg_color="black", fg_color="#1E1E1E",
                                    border_color="#3E3E3E", border_width=1, text_color="#2DF60D", justify="center")
                    elif data_list[i][j] == "FAIL":
                        self.Entry = ctk.CTkEntry(root, width=widthColumns, height=23, corner_radius=2,
                                    font=('Lato',13,'bold'), bg_color="black", fg_color="#1E1E1E",
                                    border_color="#3E3E3E", border_width=1, text_color="red", justify="center")
                    else:
                        self.Entry = ctk.CTkEntry(root, width=widthColumns, height=23, corner_radius=2,
                                    font=('Lato',13), bg_color="black", fg_color="#1E1E1E",
                                    border_color="#3E3E3E", border_width=1, text_color="white", justify="center")
                    self.Entry.grid(row=i, column=j)
                    self.Entry.insert(END, data_list[i][j])
                    self.Entry.configure(state="readonly")

    root = ctk.CTk()
    # root.title(ticketName)
    Table(root)
    root.mainloop()


from __future__ import absolute_import
from __future__ import division

import torch

from laia.utils.symbols_table import SymbolsTable

def unigram_phoc(sequence, unigram_map, unigram_levels, ignore_missing=False):
    r"""Compute the Pyramid of Histograms of Characters (PHOC) of a given
    sequence of characters (or arbitrary symbols).

    Args:
      sequence (list, tuple, str): sequence of characters.
      unigram_map (dict): map from symbols to positions in the histogram.
      unigram_levels (list): list of levels in the pyramid.
      ignore_missing (bool): If True, will ignore elements the sequence not
         present in the unigram_map dictionary.

    Returns:
      A tuple representing the PHOC of the given sequence.
    """

    def occupancy(i, n):
        return i / n, (i + 1) / n

    def overlap(a, b):
        return max(a[0], b[0]), min(a[1], b[1])

    def size(o):
        return o[1] - o[0]

    # Initialize histogram to 0
    phoc_size = len(unigram_map) * sum(unigram_levels)
    phoc = [0] * phoc_size

    # Offset of each unigram level starts in the PHOC array.
    level_offset = [0]
    for i, level in enumerate(unigram_levels[1:], 1):
        level_offset.append(
            level_offset[i - 1] + unigram_levels[i - 1] * len(unigram_map)
        )

    # Compute PHOC
    num_chars = len(sequence)
    missing_count = {}  # This is not being used.
    for i, ch in enumerate(sequence):
        if ch not in unigram_map:
            if ignore_missing:
                missing_count[ch] = missing_count.get(ch, 0) + 1
                continue
            else:
                raise KeyError(
                    "Unigram {!r} was not found in the unigram map".format(ch)
                )
        ch_occ = occupancy(i, num_chars)
        for j, level in enumerate(unigram_levels):
            for region in range(level):
                region_occ = occupancy(region, level)
                if size(overlap(ch_occ, region_occ)) / size(ch_occ) >= 0.5:
                    # Total offset in the histogram for the current level,
                    # region and character.
                    z = level_offset[j] + region * len(unigram_map) + unigram_map[ch]
                    phoc[z] = 1
    return tuple(phoc)

def new_unigram_phoc(sequence, unigram_map, unigram_levels, ignore_missing=False):
    # Initialize histogram to 0
    phoc_size = len(unigram_map) * sum(unigram_levels)
    phoc = [0] * phoc_size

    # Offset of each unigram level starts in the PHOC array.
    level_offset = [0]
    for i, level in enumerate(unigram_levels[1:], 1):
        level_offset.append(
            level_offset[i - 1] + unigram_levels[i - 1] * len(unigram_map)
        )

    # Compute PHOC
    for i, ch in enumerate(sequence):
        if ch not in unigram_map:
            if ignore_missing:
                continue
            else:
                raise KeyError(
                    "Unigram {!r} was not found in the unigram map".format(ch)
                )

        for j, level in enumerate(unigram_levels):
            r = int(i * level / len(sequence))
            z = level_offset[j] + r * len(unigram_map) + unigram_map[ch]
            phoc[z] = 1

    return tuple(phoc)

class TextToPHOC(object):
    def __init__(self, syms, levels):
        assert isinstance(syms, (dict, SymbolsTable))
        assert isinstance(levels, (list, tuple))
        self._syms = syms
        self._levels = levels

    def __call__(self, x):
        return torch.tensor(unigram_phoc(x, self._syms, self._levels))


from scrapy import Spider, Request
from crms.items import CrmsItem
import re

class CrmsSpider(Spider):
    name = 'crms_spider'
    allowed_domains = ['www.g2.com']
    start_urls = ['https://www.g2.com/categories/crm?order=g2_score#product-list']

    def parse(self, response):
        total = int(response.xpath('//div[@class="ws-nw"]/strong/text()').extract_first())
        per_page = len(response.xpath('//div[@class="paper pt-half pb-0 my-1"]'))
        total_pages = total // per_page + 1

        # List comprehension to construct all the urls
        result_urls = ['https://www.g2.com/categories/crm?order=g2_score&page={}#product-list'.format(x) for x in range(1,total_pages+1)]

        # Yield the requests to different search result urls,
        # using parse_result_page function to parse the response.
        for url in result_urls:
            yield Request(url=url, callback=self.parse_result_page)

    def parse_result_page(self, response):
        # We are looking for url of the detail page.
        detail_urls = response.xpath('//div[@class="product-listing__title"]/div/div/a[1]/@href').extract()

        # Yield the requests to the details pages,
        # using parse_detail_page function to parse the response.

        for url in detail_urls:
            yield Request(url='https://www.g2.com' + url, callback=self.parse_detail_page)

    # another method

    def parse_detail_page(self, response):

        CRM = response.xpath('//h4[@class="font-weight-bold color-dark-blue word-wrap-word"]/text()').extract()
        Description = response.xpath('//p[@itemprop="description"]/text()').extract()
        OverallRating = response.xpath('//div[@class="mr-4th border px-4th color-primary font-weight-bold border-radius--low bg-white h6"]/text()').extract()
        TotalReviewCount = response.xpath('//div[@class="as-fe"]/text()').extract()
        Review = response.xpath('//h3[@class="review-list-heading m-0 mb-half word-break-word"]/text()').extract()
        Stars = response.xpath('//span[@itemprop="reviewRating"]/meta[3]/@content').extract()
        Date = response.xpath('//div[@class="time-stamp text-small pl-4th ws-nw"]/span/time/text()').extract()

        item = CrmsItem()
        item['CRM'] = CRM
        item['Description'] = Description
        item['OverallRating'] = OverallRating
        item['TotalReviewCount'] = TotalReviewCount
        item['Review'] = Review
        item['Stars'] = Stars
        item['Date'] = Date

        yield item


import os
import shutil
from flask import render_template,request, redirect, send_file, session, url_for, make_response
from app import app, CVATapi, comparedb, dbquery, truthdb, predictdb
from dotenv import load_dotenv
from queue import Queue
from threading import Thread

load_dotenv()

url_cvat = os.getenv('URL_CVAT')

## PRIMA DI OGNI RICHIESTA FACCIO QUESTO CONTROLLO ##
#####################################################################################################

@app.before_request
def before_request():

    # controllo che il server cvat sia in run
    #status = CVATapi.check_serverCVAT_connection()
    #if status == 'Server not connect':
        #return make_response(render_template("serverError.html", info="Server non connesso"), 404)

    # controllo che i progetti cvat esistenti nel db siano esistenti anche su cvat (in questo modo
    # eventuali progetti eliminati lato cvat saranno eliminati anche lato db)
    # p_cvat contiene i progetti esistenti su cvat, a prescindere da MVE
    # ps_db contiene i progetti cvat esistenti sul db
    projectsCVAT = CVATapi.get_projects_id()
    projectsCVATdb = dbquery.get_projectsCVAT_id()

    for p in projectsCVATdb:
        if p not in projectsCVAT:
            dbquery.delete_projectCVAT(p)

## HOME ##
#####################################################################################################

# home page con la vista dei progetti (MVE e CVAT)
@app.route("/")
@app.route("/index")
def index():

    # prendo tutti i progetti MVE e CVAT
    projectsMVE = dbquery.get_projectsMVE()
    projectsCVAT =  dbquery.get_projectMVExProjectCVAT()
    predictions = dbquery.get_predList(id)

    return render_template('index.html', projectsMVE=projectsMVE, projectsCVAT=projectsCVAT, predictions=predictions, url_cvat=url_cvat)

## GESTIONE PROGETTI MVE ##
#####################################################################################################

# CREAZIONE di un nuovo progetto MVE
@app.route("/new_projectMVE")
def new_projectMVE():
    return render_template('new_projectMVE.html')

# Ricezione dei dati per la creazione di un nuovo progetto MVE
@app.route("/new_projectMVE_data", methods=['POST', 'GET'])
def new_projectMVE_data():

    if request.method == "POST":
        name_prj = request.form['name-project']
        desc_prj = request.form['desc-project']
        img_prj = request.files['img-project']
        dbquery.new_projectMVE(name_prj,desc_prj, img_prj)

        return redirect(url_for('index'))
    return ('',204)

# MODIFICA di un nuovo progetto MVE
@app.route("/edit_projectMVE/<id>")
def edit_projectMVE(id):
    data = dbquery.get_projectMVE(id)
    if (data == []):
        return make_response(render_template("404.html", info="Il progetto {} non esiste".format(id)), 404)

    predictions = dbquery.get_predList_by_id(id)

    return render_template('edit_projectMVE.html', id=id, data=data, predictions=predictions)

# Ricezione dei dati per la modifica di un nuovo progetto MVE
@app.route("/edit_projectMVE_data", methods=['POST', 'GET'])
def edit_projectMVE_data():

    if request.method == "POST":
        id_prj = request.form['id-project']
        name_prj = request.form['name-project']
        desc_prj = request.form['desc-project']
        img_prj = request.files['img-project']

        dbquery.edit_projectMVE(id_prj, name_prj,desc_prj, img_prj)

        return redirect(url_for('index'))
    return ('',204)

# ELIMINAZIONE di un progetto MVE
@app.route("/delete_projectMVE/<id>")
def delete_projectMVE(id):

    # controllo che il progetto mve esista
    project = dbquery.get_projectMVE(id)
    if (project == []):
        return make_response(render_template("404.html", info="Il progetto {} non esiste".format(id)), 404)

    # prendo i progetti cvat associati al progetto mve da eliminare
    prjsCVAT = dbquery.get_projectsCVAT(id)
    dbquery.delete_projectMVE(id)
    dbquery.delete_MVSxCVAT_MVE(id)

    # se esistono elimino ogni progetto cvat
    if (len(prjsCVAT) != 0):
        for pCVAT in prjsCVAT:
            CVATapi.delete_project(pCVAT[1])

    return redirect(url_for('index'))

## GESTIONE PROGETTI CVAT ##
#####################################################################################################

# VISUALIZZAZIONE di un progetto CVAT
@app.route("/project/<id>")
def project(id):

    # controlliamo che il progetto esista
    project = dbquery.get_projectCVAT(id)

    # otteniamo tutti i progetti per poter cambiare la scelta del progetto corrente dalla pagina project.html
    projectsMVE = dbquery.get_projectsMVE()
    projectsCVAT =  dbquery.get_projectMVExProjectCVAT()

    if (project == []):
        return render_template('project.html', id=id, projectsMVE=projectsMVE, projectsCVAT=projectsCVAT, info="progetto non esistente")

    # l'api di cvat ritorna solo 10 task (in questo caso gli ultimi)
    tasks = CVATapi.get_tasks(id)

    return render_template('project.html', id=id, projectsMVE=projectsMVE, projectsCVAT=projectsCVAT, tasks=tasks, info="", url_cvat=url_cvat)

# CREAZIONE di un nuovo progetto CVAT, dato un progetto MVE
@app.route("/new_projectCVAT/<id_prj_MVE>")
def new_projectCVAT(id_prj_MVE):

    # controllo che il progetto mve esista
    project = dbquery.get_projectMVE(id_prj_MVE)
    if (project == []):
        return make_response(render_template("404.html", info="Il progetto {} non esiste".format(id_prj_MVE)), 404)

    name_prj_MVE = request.args.get('name_prj_MVE')

    return render_template('new_projectCVAT.html', id_prj_MVE=id_prj_MVE, name_prj_MVE=name_prj_MVE)

# Ricezione dei dati per la creazione di un nuovo progetto CVAT
@app.route("/new_projectCVAT_data", methods=['POST', 'GET'])
def new_projectCVAT_data():
    if request.method == "POST":
        name_prj = request.form['name-project']
        id_prj_MVE = request.form['id-prj-MVE']

        id_prj_CVAT = CVATapi.create_project(name_prj)

        dbquery.new_projectCVAT(id_prj_MVE, id_prj_CVAT)

        return redirect(url_for('index'))
    return ('',204)

# ELIMINAZIONE di un progetto CVAT
@app.route("/delete_projectCVAT/<id>")
def delete_projectCVAT(id):

    # controllo che il progetto mve esista
    project = dbquery.get_projectCVAT(id)
    if (project == []):
        return make_response(render_template("404.html", info="Il progetto {} non esiste".format(id)), 404)

    tasks = CVATapi.get_tasks_ids(id)
    for task in tasks:
        dbquery.delete_MVSxCVAT_task(task)

    dbquery.delete_projectCVAT(id)
    CVATapi.delete_project(id)

    return redirect(url_for('index'))

## TASK MANAGEMENT - THREAD ##
#####################################################################################################

# La funzione prende dalla coda il primo task inserito poi in un ciclo for scorre tutte le richieste
# finche non trova qeulla con il task corrispondente. a quel punto prende i dati (uuid, nametask, id)
# e genera n task cvat con le immagini caricate
def worker(queue):
    while True:
        # get the next task, blocking until one is available.
        task = queue.get()
        # The app is available to the worker, but be wary of
        # using app services without locking.
        ##### WORK TO DO IN BACKGROUND #####

        for request in requests:
            if (request['task']==task):
                uuid = request['uuid']
                name_task = request['name_task']
                id = request['id_prjCVAT']
                keyLogin = CVATapi.generate_key_login()

                idMVE = dbquery.get_projectCVAT(id)[0]

                directory = 'temp{}'.format(uuid)
                dir = os.listdir(directory)
                current_task = 1

                # Checking if the list is empty or not
                while len(dir) != 0:
                    taskId = CVATapi.create_empty_task(id, name_task, current_task, keyLogin)

                    fs = CVATapi.select_images(uuid, idMVE, taskId)

                    CVATapi.upload_images(uuid, fs, keyLogin, taskId)
                    current_task = current_task + 1
                    dir = os.listdir(directory)

                shutil.rmtree(directory)
        for request in requests.copy():
            if (request['task']==task):
                requests.remove(request)

                #duration = app.config.get('WORK_TIME', 3)

                #sleep(duration)  # simulate doing work
                print("did task {!r}".format(task))

# Give the worker thread a Queue that's shared with the app, and start
# the worker. The worker will block immediately until a task is added
# to the queue.
work_queue = Queue()
worker_thread = Thread(target=worker, args=(work_queue,), daemon=True)
worker_thread.start()
requests = []
task = 0

## CARICAMENTO IMMAGINI - CREAZIONE DI TASK - ELIMINAZIONE TASK ##
#####################################################################################################

# CARICAMENTO immagini
@app.route('/upload/<id>')
def upload(id):
    # controllo che il progetto sia esistente
    project = dbquery.get_projectCVAT(id)
    if (project == []):
        return make_response(render_template("404.html", info="Il progetto {} non esiste".format(id)), 404)

    return render_template('upload_img.html', id=id)

@app.route('/get_regex_file', methods=['GET', 'POST'])
def get_regex_file():
    if request.method == "POST":
        data = request.get_json()
        print(data['uuid'])
        print(data['data'])

    session['regex'+data['uuid']] = data['data']

    return "ok"
# Caricamento effettivo delle immagini e creazione di n task a seconda del peso totale delle immagini
# NB. questa rotta viene raggiunta quando viene eseguito il submit del form presente alla pagina upload_img.html;
# ogni volta che viene fatto un submit viene incrementato il numero di task (niente a che fare con cvat, in qeusto caso è
# una variabile globale) e viene creata una istanza this_request con i dati della richiesta corrente con anche il numero del task
# corrispondente. questa istanza viene aggiunta ad una lista di richieste (requests). viene creato un nuovo thread che chiama
# la funzione "warker" con parametro una coda (work_queue) di task. Vedi funzione per vedere cosa succede
@app.route('/uploader/<id>', methods=['POST', 'GET'])
def uploader(id):
    this_name_task = request.form['name-task']
    files = request.files.getlist('fileList')
    this_uuid = request.form.get('uuid')
    regex = session['regex'+this_uuid]
    regex_files = []
    for file in files:
        if file.filename in regex:
            regex_files.append(file)
    session.pop('regex'+this_uuid)

    totalSize = CVATapi.get_files(regex_files, this_uuid)
    global task
    task += 1
    this_request = {
        'task': task,
        'uuid': this_uuid,
        'name_task': this_name_task,
        'id_prjCVAT': id
    }
    global requests
    requests.append(this_request)
    print("add task {!r}".format(task))
    work_queue.put(task)
    return redirect(url_for('project', id=id))

# ELIMINAZIONE di un task
@app.route("/delete_task/<id>")
def delete_task(id):

    prj_id = request.args.get('prj_id')
    CVATapi.delete_task(id)
    dbquery.delete_MVSxCVAT_task(id)

    return redirect(url_for('project', id=prj_id))

## VERITA ##
#####################################################################################################

# NB l'id che passo come parametro è l'id del progetto MVE scelto

# CARICAMENTO truth
@app.route("/upload_truth/<id>")
def upload_truth(id):
    # controllo che il progetto mve esista
    project = dbquery.get_projectMVE(id)
    if (project == []):
        return make_response(render_template("404.html", info="Il progetto {} non esiste".format(id)), 404)

    return render_template('upload_truth.html', id=id)

# Caricamento effettivo del file e aggiunta di istanze nelle tabelle Truth e TruthValues
@app.route('/uploader_truth/<id>', methods=['POST', 'GET'])
def uploader_truth(id):
    file_truth = request.files['file-truth']
    resp = truthdb.read_file(file_truth,id)
    session['duplicates'] = resp

    return redirect(url_for('upload_truth', id=id))

# SCARICAMENTO verita
@app.route('/download_truth/<id>', methods=['POST', 'GET'])
def download_truth(id):
    truthdb.download_truth(id)
    namefile = truthdb.download_truth(id)
    return send_file("../"+namefile, as_attachment=True)

## PREDIZIONI ##
#####################################################################################################

# NB l'id che passo come parametro è l'id del progetto MVE scelto

# CARICAMENTO predizione
@app.route("/upload_prediction/<id>")
def upload_prediction(id):
    # controllo che il progetto mve esista
    project = dbquery.get_projectMVE(id)
    if (project == []):
        return make_response(render_template("404.html", info="Il progetto {} non esiste".format(id)), 404)

    return render_template('upload_prediction.html', id=id)

# Caricamento effettivo del file e aggiunta di istanze nelle tabelle Truth e TruthValues
@app.route('/uploader_prediction/<id>', methods=['POST', 'GET'])
def uploader_prediction(id):
    file_truth = request.files['file-pred']

    return predictdb.read_file(file_truth,id)

# ELIMINAZIONE di una predizione
@app.route("/delete_prediction/<id>")
def delete_prediction(id):

    dbquery.delete_prediction(id)
    prj_id = request.args.get('prj_id')

    return redirect(url_for('edit_projectMVE', id=prj_id))

# SCARICAMENTO predizione
@app.route('/download_pred/<id>', methods=['POST', 'GET'])
def download_pred(id):
    if request.method == "POST":
        pred = request.form['select-prediction']
        namefile = predictdb.download_pred(id, pred)
        return send_file("../"+namefile, as_attachment=True)
    return redirect(url_for('index'))

## CONFRONTO PREDIZIONI/VERITA  ##
#####################################################################################################

state_compare = False

# pagina principale
@app.route("/compare/<id>")
def compare(id):
    comparisons = comparedb.get_comparisons(id)
    predictions = dbquery.get_predList_by_id(id)
    truth = dbquery.get_truth_mve(id)
    dname = session.get('dname', None)
    session.pop('dname', None)
    info = session.get('info', None)
    session.pop('info', None)
    status = session.get('status', None)
    session.pop('status', None)

    return render_template('compare.html', id=id, predictions=predictions, comparisons=comparisons, truth=truth, dname=dname, info=info, status=status)

# Visualizza predizione
@app.route("/view_prediction/<id>",  methods=['POST', 'GET'])
def view_prediction(id):
    if request.method == "POST":
        pred = request.form['select-prediction']

        global state_compare
        if state_compare == False:
            state_compare = True
            database = comparedb.view_prediction(id, pred)
            session['dname'] = database.name
            state_compare = False
        else:
            session['info'] = "Server occupato. riprova tra qualche secondo."
        session['status']="pred"

    return redirect(url_for('compare', id=id))

# Confronto predizioni
@app.route("/compare_predictions/<id>",  methods=['POST', 'GET'])
def compare_predictions(id):
    if request.method == "POST":
        pred1 = request.form['select-first-prediction']
        pred2 = request.form['select-second-prediction']
        global state_compare
        if state_compare == False:
            state_compare = True
            database = comparedb.compare_predictions(id, pred1, pred2)
            session['dname'] = database.name
            state_compare = False
        else:
            session['info'] = "Server occupato. riprova tra qualche secondo."
        session['status']="pred-vs-pred"

    return redirect(url_for('compare', id=id))

# Confronto predizione vs verita
@app.route("/compare_pred_truth/<id>",  methods=['POST', 'GET'])
def compare_pred_truth(id):
    if request.method == "POST":
        pred = request.form['select-prediction']
        global state_compare
        if state_compare == False:
            state_compare = True
            database = comparedb.compare_pred_truth(id, pred)
            session['dname'] = database.name
            state_compare = False
        else:
            session['info'] = "Server occupato. riprova tra qualche secondo."
        session['status']="pred-vs-truth"

    return redirect(url_for('compare', id=id))

# ELIMINAZIONE di un confronto
@app.route("/delete_compare/<id>")
def delete_compare(id):
    dir_name = request.args.get('dir_name')
    shutil.rmtree('datasets/'+dir_name)
    return redirect(url_for('compare', id=id))

# Visualizza qualcosa di gia esistente
@app.route("/compare_existing/<id>",  methods=['POST', 'GET'])
def compare_existing(id):
    pred1 = request.args.get('pred1')
    tab = request.args.get('pred2')
    global state_compare
    if state_compare == False:
        state_compare = True
        if (tab == "/"):
            database = comparedb.view_prediction(id, pred1)
        elif (tab == "Truth"):
            database = comparedb.compare_pred_truth(id, pred1)
        else:
            database = comparedb.compare_predictions(id, pred1, tab)
        state_compare = False
        session['dname'] = database.name
    else:
        session['info'] = "Server occupato. riprova tra qualche secondo."

    return redirect(url_for('compare', id=id))

## GESTIONE ERRORI  ##
#####################################################################################################

@app.errorhandler(404)
def page_not_found(error):
    return render_template('404.html', info="Pagina non trovata"), 404


# -*- coding: utf-8 -*-

from datetime import datetime, timedelta
import itertools

from ask.models import Question
from contracts import contract
from django.conf import settings
from django.core.exceptions import ValidationError
from django.core.urlresolvers import reverse
from django.core.validators import validate_comma_separated_integer_list
from django.db import models
from django.db.models import Count
from django.apps import apps
from django.template import Context, loader
from signalbox.models import Reply, Answer
from signalbox.models.validators import is_24_hour, only_includes_allowed_fields
from signalbox.utils import incomplete, in_range
# from stats.stats import chisquare
from signalbox.s3 import CustomS3BotoStorage
from yamlfield.fields import YAMLField

User = settings.AUTH_USER_MODEL

class StudySite(models.Model):
    """Location (for multicenter studies) at which participants recruited."""

    name = models.CharField(max_length=500,
                            help_text="""e.g. 'Liverpool', 'London' """)

    class Meta:
        app_label = 'signalbox'

    def __unicode__(self):
        return self.name

class StudyManager(models.Manager):
    def get_by_natural_key(self, slug):
        return self.get(slug=slug)

class Study(models.Model):
    """Details of a study (or substudy or a large trial)."""

    objects = StudyManager()

    def natural_key(self):
        return (self.slug, )

    auto_randomise = models.BooleanField(default=True,
                                         help_text="""If on, then users are automatically added to a study
            condition when a new membership is created.""")

    auto_add_observations = models.BooleanField(default=True,
            help_text="""If on, and auto_randomise is also on, then observations will
            be created when a membership is created. Has no effect if
            auto_randomise is off.""")

    visible = models.BooleanField(default=False,
        help_text="""Controls the display of the study on the /studies/ page.""", db_index=True)

    paused = models.BooleanField(default=False,
        help_text="""If True, pauses sending of signals. Observations missed
        will not be caught up later without manual intervention.""", db_index=True)

    participants = models.ManyToManyField(settings.AUTH_USER_MODEL, through='Membership', )

    slug = models.SlugField(help_text="""A short name
        used to refer to the study in URLs and other places.""")

    name = models.CharField(max_length=200)

    study_email = models.EmailField(blank=False,
                                    help_text="The return email address for all mailings/alerts.")

    twilio_number = models.ForeignKey('twiliobox.TwilioNumber', blank=True, null=True)

    ad_hoc_askers = models.ManyToManyField(
        'ask.Asker', blank=True,
        help_text="""Questionnaires which can be completed ad-hoc.""",
        verbose_name="Questionnaires available ad-hoc")

    ad_hoc_scripts = models.ManyToManyField(
        'signalbox.Script', blank=True,
        help_text="""Scripts which can be initiated by users on an ad-hoc basis.
        IMPORTANT. Scripts selected here will appear on the 'add extra data' tab
        of participants' profile pages, and will display the name of the study,
        and the name of the script along with the username of any related
        membership. BE SURE THESE DO NOT LEAK INFORMATION you would not which
        participants to see.""",
        verbose_name="""scripts allowed on an ad-hoc basis""")

    createifs = models.ManyToManyField(
        'signalbox.ObservationCreator', blank=True,
        help_text="""Rules by which new observations might be made in response
        to participant answers.""", verbose_name="""Rules to create new
        observations based on user responses""")

    blurb = models.TextField(blank=True, null=True, help_text="""Snippet of
        information displayed on the studies listing page.""")

    study_image = models.ImageField(blank=True, null=True,
        storage=CustomS3BotoStorage(),
        upload_to="study/images/")

    briefing = models.TextField(blank=True, null=True,
        help_text="""Key information displayed to participants before joining
        the study.""")

    consent_text = models.TextField(blank=True, null=True,
        help_text="""User must agree to this before entering study.""")

    welcome_text = models.TextField(blank=True, null=True,
        default="""Welcome to the study""",
        help_text="""Text user sees in a message box when they signup for the
        the study. Note his message is not needed if participants will be
        added to studies by the experimenter, rather than through the
        website.""")

    max_redial_attempts = models.IntegerField(default=3,
        help_text='''Only relevant for telephone calls: maximium
        number of times to try and complete the call. Default
        if nothing is specified is 3''')

    redial_delay = models.IntegerField(default=30,
        help_text='''Number of minutes to wait before calling again (mainly for phone calls)''')

    working_day_starts = models.PositiveIntegerField(default=8,
        validators=[is_24_hour], help_text="""Hour (24h clock) at which phone
            calls and texts should start to be made""")

    working_day_ends = models.PositiveIntegerField(default=22,
       validators=[is_24_hour], help_text="""Hour (24h clock) at which phone
            calls and texts should stop.""")

    def in_working_hours(self, hour=None):
        """Return True if we are within the study's working hours.

        Working hours are defined by the `working_day_starts` and
        `working_day_ends` fields."""

        hour = hour or datetime.now().time().hour
        workinghours = list(range(
            self.working_day_starts, self.working_day_ends - 1))
        if hour in workinghours:
            return True
        return False

    show_study_condition_to_user = models.BooleanField(default=False,
        help_text="""If True, the user will be able to see their condition on
        their homepage. Useful primarily for experiments where participants
        signs up with the experimenter present and where blinding is not a
        concern (e.g. where the experimenter carries out one of several
        manipulations).""")

    randomisation_probability = models.DecimalField(
        max_digits=2, decimal_places=1, default=.5,
        help_text="""Indicates the probability that the adaptive algorithm will
        choose weighted randomisation rather than allocation to the group which
        minimises the imbalance (minimisation or adaptive randomisation).
        For example, if set to .5, then deterministic allocation to the
        smallest group will only happen half of the time. To turn off adaptive
        randomisation set to 1.""")

    visible_profile_fields = models.CharField(max_length=200,
        blank=True, null=True, help_text="""Available profile fields
        for this study. Can be any of: {0}, separated by a space.""".format(
        ", ".join(settings.USER_PROFILE_FIELDS)),
        validators=[only_includes_allowed_fields])

    required_profile_fields = models.CharField(max_length=200,
        blank=True, null=True, help_text="""Profile fields which users
        will be forced to complete for this study. Can be any of: {0},
        separated by a space.""".format(", ".join(settings.USER_PROFILE_FIELDS)),
        validators=[only_includes_allowed_fields])

    valid_telephone_country_codes = models.CharField(
        validators=[validate_comma_separated_integer_list],
        max_length=32, default="44",
        help_text="""Valid national dialing codes for participants in this study.""")

    def profile_fields_dict(self):
        vis = self.visible_profile_fields or ""
        req = self.required_profile_fields or ""
        return {
            'visible': set(vis.split() + req.split()),
            'required': req.split()
        }

    def randomised(self):
        """Return the Memberships which have been randomised to a Condition in this Study."""

        return self.membership_set.filter(date_randomised__isnull=False)

    def unrandomised(self):
        """Return a Queryset of participants NOT randomised to a Condition."""

        return self.membership_set.filter(date_randomised__isnull=True)

    def scripts_used_by_this_study(self):
        return set(
            itertools.chain(
                *[i.scripts.all() for i in self.studycondition_set.all()])
        )

    def askers_used_by_this_study(self):
        askers = itertools.chain(*[[i.asker for i in j.scripts.all() if i.asker]
                                   for j in self.studycondition_set.all()])
        return set(askers)

    def questions_used(self):
        """Return all :class:`Question`s used to record an :class:`Answer` for this class:`Study`."""

        answers = Answer.objects.filter(reply__observation__dyad__study=self)
        questions = [i.question for i in answers if i.question]

        return list(set(questions))

    def observations_with_duplicate_replies(self):
        Observation = apps.get_model('signalbox', 'Observation')
        obs = Observation.objects.filter(dyad__study=self)
        dupes = obs.annotate(
            num_replies=Count('reply')).filter(num_replies__gt=1)
        return dupes

    def unresolved_observations_with_duplicate_replies(self):
        obs = self.observations_with_duplicate_replies()
        unresolved = [i for i in obs if i.reply_set.filter(
            is_canonical_reply=True).count() == 0]
        return unresolved

    def observations(self):
        Observation = apps.get_model('signalbox', 'Observation')
        return Observation.objects.filter(dyad__study=self)

    class Meta:
        app_label = 'signalbox'
        ordering = ["name"]
        verbose_name_plural = "studies"

    # @contract
    # def chisq_imbalance(self):
    #     """Returns the chi2 and p value for the imbalance in allocations.

    #     That is, between current membership allocations and the expected
    #     frequencies based on the StudyCondition weights.

    #     :rtype: tuple(float, float)
    #     """
    #     conditions = self.studycondition_set.all()
    #     counts = [i.users.count() for i in conditions]
    #     expected = [i.expected_n() for i in conditions]

    #     return chisquare(counts, expected)

    def get_absolute_url(self):
        return reverse('study', args=(self.pk,))

    def clean(self, *args, **kwargs):

        if self.auto_randomise is False and self.auto_add_observations is True:
            raise ValidationError("""Observations can only be added automatically
                when participants are automatically randomised to a condition.""")

        needtwilio = bool(sum([x.script_type.require_study_ivr_number for x in self.scripts_used_by_this_study()]))
        if needtwilio and not self.twilio_number:
            raise ValidationError("This study uses Scripts which will make calls and require Twilio account details.")

        super(Study, self).clean(*args, **kwargs)

    def __unicode__(self):
        return self.name

class StudyCondition(models.Model):
    '''Groupings of participants within a study.'''
    study = models.ForeignKey('signalbox.Study', blank=True, null=True)
    users = models.ManyToManyField(settings.AUTH_USER_MODEL, through='Membership')
    tag = models.SlugField(max_length=255, default="main")
    display_name = models.CharField(max_length=30, blank=True, null=True,
                                    help_text="""A label for this condition which can be shown to
        the Participant (e.g. a non-descriptive name used to identify a
        condition in an experimental session without breaking a blind.""", )
    weight = models.IntegerField(default=1,
                                 help_text="""Relative weights to allocate users to conditions""")
    scripts = models.ManyToManyField('signalbox.Script', blank=True)
    metadata = YAMLField(blank=True, null=True,
        help_text="""YAML meta data available describing this condition. Can be used on Questionnaires,
        e.g. to conditionally display questions.""")

    def expected_n(self):
        """Return the expected number of participants based on the total currently allocated to the parent study."""
        weights = [i.weight for i in self.study.studycondition_set.all()]
        w = self.weight / sum(weights)
        return self.study.membership_set.count() * w

    class Meta:
        ordering = ['study', 'tag']
        app_label = 'signalbox'

    def __unicode__(self):
        return "%s > %s " % (self.study.name, self.tag)


from __future__ import annotations

__ALL__ = ["WaterQualityIndex"]

from typing import Union, ClassVar, Optional

from django.db import models
from django.utils.translation import gettext_lazy as _

from gcampus.core.models import Measurement
from gcampus.core.renderable import Renderable

class WaterQualityIndex(models.Model, Renderable):
    class Meta:
        abstract = True

    slug: ClassVar[str]  # subclasses must set 'slug'
    icon_name: ClassVar[str]  # subclasses must set 'icon_name'
    template_name: ClassVar[str] = "gcampuscore/components/index_card.html"
    validity_warning = 0.7
    validity_limit = 0.4

    measurement: Union[models.ForeignKey, Measurement]
    value: models.FloatField = models.FloatField(
        default=0,
        null=True,
        blank=True,
        verbose_name=_("Value"),
    )

    classification: models.CharField = models.CharField(
        max_length=10,
        null=True,
        blank=True,
        verbose_name=_("Classification"),
    )

    description: models.CharField = models.CharField(
        max_length=50,
        null=True,
        blank=True,
        verbose_name=_("Description"),
    )

    validity: models.DecimalField = models.DecimalField(
        max_digits=3,
        decimal_places=2,
        default=0,
        verbose_name=_("Validity"),
    )

    def update(self, commit=True):
        self._update_value(commit=False)
        self._update_classification(commit=False)
        self._update_description(commit=False)
        self._update_validity(commit=False)
        if commit:
            self.save()

    def _update_value(self, commit=True):
        parameters = self.measurement.parameters.all()
        kwargs = dict()
        for p in parameters:
            kwargs[p.parameter_type.identifier] = p.value
        self.value = self.calculate_index(**kwargs)
        if commit:
            self.save()

    def _update_classification(self, commit=True):
        self.classification = self.calculate_classification(self.value)
        if commit:
            self.save()

    def _update_description(self, commit=True):
        self.description = self.calculate_description(self.value)
        if commit:
            self.save()

    def _update_validity(self, commit=True):
        parameters = list(
            self.measurement.parameters.values_list(
                "parameter_type__identifier", flat="true"
            )
        )
        self.validity = self.calculate_validity(parameters)
        if commit:
            self.save()

    @property
    def valid_flow_type(self) -> bool:
        raise NotImplementedError("subclasses must implement valid_flow_type")

    @classmethod
    def calculate_index(cls, **kwargs) -> float:
        raise NotImplementedError()

    @classmethod
    def calculate_classification(cls, value) -> Union[None, str]:
        raise NotImplementedError()

    @classmethod
    def calculate_description(cls, value) -> Union[None, str]:
        raise NotImplementedError()

    @classmethod
    def calculate_validity(cls, parameters) -> float:
        raise NotImplementedError()

    @property
    def has_validity_warning(self) -> bool:
        """Show a warning if the validity is too low"""
        return self.validity <= self.validity_warning

    @property
    def show_classification(self) -> bool:
        """Show the classification (and value) based on the threshold
        :attr:`.validity_limit`"""
        return self.validity >= self.validity_limit

    def get_indicator_template(self) -> Optional[str]:
        """Template for the indicator list"""
        return f"gcampuscore/components/indicator_lists/{self.slug!s}.html"

    def get_css_class(self) -> str:
        """CSS class for changing the font color based on the index
        classification"""
        return f"{self.slug!s}-class-{self.classification!s}"

    def get_icon_template(self) -> str:
        return f"gcampuscore/icons/{self.icon_name!s}.html"

    def get_context(self, **kwargs) -> dict:
        context: dict = {
            "name": self._meta.verbose_name,
            "value": self.value,
            "validity": self.validity,
            "classification": self.classification,
            "description": self.description,
            "has_validity_warning": self.has_validity_warning,
            "show_classification": self.show_classification,
            "indicator_template": self.get_indicator_template(),
            "icon_template": self.get_icon_template(),
            "css_class": self.get_css_class(),
            "measurement": self.measurement,
        }
        context.update(kwargs)
        return context


from __future__ import annotations

from pprint import pprint
from typing import Any, Optional, Union

import datetime
import pickle
import os.path

from google.auth.transport.requests import Request
from google.oauth2.service_account import Credentials
from google_auth_oauthlib.flow import InstalledAppFlow
from googleapiclient.discovery import Resource, build

from .explore_md_file import parse_events
from .config import CALENDAR_ID
from .colors import color_text
from .logger import logger
from .model import Event

# If modifying these scopes, delete the file token.pickle.
SCOPES = ["https://www.googleapis.com/auth/calendar"]

def get_credentials() -> Union[Credentials, Any]:
    """
    Returns the credentials for google calendar api

    @return service: (googleapiclient.discovery.Resource) le service passé
    """
    creds = None
    # The file token.pickle stores the user's access and refresh tokens/ and is
    # created automatically when the authorization flow completes for the first
    # time.
    if os.path.exists("tokens/token.pickle"):
        with open("tokens/token.pickle", "rb") as token:
            creds = pickle.load(token)
    # If there are no (valid) credentials available, let the user log in.
    if not creds or not creds.valid:
        if creds and creds.expired and creds.refresh_token:
            creds.refresh(Request())
        else:
            flow = InstalledAppFlow.from_client_secrets_file(
                "tokens/credentials.json", SCOPES
            )
            creds = flow.run_local_server(port=0)
        # Save the credentials for the next run
        with open("tokens/token.pickle", "wb") as token:
            pickle.dump(creds, token)
    return creds

def build_service() -> Resource:
    """
    Return the google api client service ressource
    with methods for interaction with the service.
    Start by building the credentials if needed.

    @return: (googleapiclient.discovery.Resource)
    """
    creds = get_credentials()
    service = build("calendar", "v3", credentials=creds)
    return service

def sync_event_from_md(
    service: Resource,
    path: str,
) -> None:
    """
    Create or update events from md file

    @param service: (Resource) the google api ressource
    @param path: (str) path to the md file
    @returns: (None)
    @SE: insert or update events for a given week
    """
    event_list = parse_events(path)
    pprint(event_list)
    for event_details in event_list:
        update_or_create_event(service, event_details)

def update_or_create_event(
    service: Resource,
    event_details: Event,
) -> None:
    """
    Sync a single event read from a .md file.
    seek if the event already exists,
    if there's already an event, it's updated.
    Else, a new one is created

    @param service: (Resource) the google api ressource
    @param event_details: (Event) the description of an event
    @returns: (None)
    """
    if not event_details.is_all_day:
        create_or_update_timed_event(service, event_details)
    else:
        create_or_update_day_event(service, event_details)

def create_or_update_timed_event(
    service: Resource,
    event_details: Event,
):
    """
    Sync a single TIMED event read from a .md file.
    seek if the event already exists,
    if there's already an event, it's updated.
    Else, a new one is created

    @param service: (Resource) the google api ressource
    @param event_details: (Event) the description of an event
    @returns: (None)
    """
    existing_event = find_timed_event_matching_time(event_details, service)
    if existing_event is None:
        create_event(
            service=service,
            event_details=event_details,
        )
    else:
        update_event(
            service=service,
            new_event=event_details,
            old_event=existing_event,
        )

def create_or_update_day_event(
    service: Resource,
    event_details: Event,
):
    """
    Sync a single DAY event read from a .md file.
    seek if the event already exists,
    if there's already an event, it's updated.
    Else, a new one is created

    @param service: (Resource) the google api ressource
    @param event_details: (Event) the description of an event
    @returns: (None)
    """
    timeMin = one_day_earlier(event_details.start["date"])
    timeMax = event_details.end["date"] + "T00:00:00Z"
    existing_events = list(retrieve_day_events_matching_date(timeMin, timeMax, service))
    if event_details in existing_events:
        update_event(
            service=service,
            new_event=event_details,
            old_event=existing_events[existing_events.index(event_details)],
        )
    else:
        create_event(service=service, event_details=event_details)

def one_day_earlier(date: str) -> str:
    """
    Shift the date back one day and format it for the API.

    @param date: (str) formated like "%Y-%m-%d"
    @returns: (str) formated like "%Y-%m-%dT00:00:00Z"
    """
    return (
        datetime.datetime.strptime(date, "%Y-%m-%d") - datetime.timedelta(days=1)
    ).strftime("%Y-%m-%dT00:00:00Z")

def retrieve_events(timeMin: str, timeMax: str, service: Resource) -> map[Event]:
    """
    Returns a generator of Events with time between timeMin and timeMax.
    @param timeMin: (str)
    @param timeMax: (str)
    @param service: (Resource)
    @returns: (map[Event]) generator of Event created on the fly.
    """
    return map(
        Event.from_dict,
        service.events()
        .list(
            calendarId=CALENDAR_ID,
            timeMin=timeMin,
            timeMax=timeMax,
            maxResults=200,
            singleEvents=True,
            orderBy="startTime",
        )
        .execute()
        .get("items", []),
    )

def find_timed_event_matching_time(
    event: Event,
    service: Resource,
) -> Optional[Event]:
    """
    Look for an event by given dates in calendar.
    If one is found, return the event.
    Else, return None.

    @param event: (Event) event instance
    @return: (Optional[Event]) Already existing event with overlaping time.
    """
    if event.is_all_day:
        return None
    timeMin = event.start["dateTime"]
    timeMax = event.end["dateTime"]

    events_filtered = list(
        filter_only_timed_events(
            retrieve_events(timeMin, timeMax, service),
        )
    )
    if events_filtered and not events_filtered[0].is_all_day:
        return events_filtered[0]

def filter_only_all_day_events(events: map[Event]) -> filter[Event]:
    """
    Filter a map of Event to only keep all day events.

    @param events: (map[Event])
    @returns: (filter[Event])
    """
    return filter(lambda event: "date" in event.start, events)

def filter_only_timed_events(events: map[Event]) -> filter[Event]:
    """
    Filter a map of Event to only keep timed events.

    @param events: (map[Event])
    @returns: (filter[Event])
    """
    return filter(lambda event: not "date" in event.start, events)

def retrieve_day_events_matching_date(
    timeMin: str,
    timeMax: str,
    service: Resource,
) -> filter[Event]:
    """
    Returns a list of day events with same start and end time.
    @param timeMin: (str) like 2022-12-31T00:00:00Z
    @param timeMax: (str) like 2022-12-31T00:00:00Z
    @param service: (Resource) the google api ressource
    @returns: (filter[Event]) a collection of filtered day events where the date matches.
    """
    return filter_only_all_day_events(
        retrieve_events(timeMin, timeMax, service),
    )

def create_event(
    service: Resource,
    event_details: Event,
) -> None:
    """
    Create a new event with given details
    @param event_details: (dict) description of the event
    @param service: (google api ressource service) the service
    @return: (None)
    """
    event = (
        service.events()
        .insert(
            calendarId=CALENDAR_ID,
            body=event_details.__dict__,
        )
        .execute()
    )

    creation_event_msg = (
        f"Event created: {event_details.readable_start_date()} {event.get('htmlLink')}"
    )
    print(color_text(creation_event_msg, "YELLOW"))
    logger.warning(creation_event_msg)

def update_event(
    service: Resource,
    new_event: Event,
    old_event: Event,
) -> None:
    """
    Update the details of an event.
    The event can be passed or only his id
    If no service is given, will create the service

    @param event_details: (dict) les attributs de l'événements à mettre à jour
    @param new_event: (Event) the new event to push
    @param old_event: (Event) the old event to update
    @returns: (None)
    """
    old_event.update(new_event)

    updated_data = (
        service.events()
        .update(
            calendarId=CALENDAR_ID,
            eventId=old_event.id,
            body=old_event.__dict__,
        )
        .execute()
    )
    update_event_msg = (
        f"Event updated: {new_event.readable_start_date()} {updated_data['htmlLink']}"
    )
    print(color_text(update_event_msg, "CYAN"))
    logger.warning(update_event_msg)


#!/usr/bin/env python

import sys
import json
import urllib
import requests
import threading
import xml.etree.ElementTree as ET

def out(s):
  sys.stdout.write(s + '\n')
  sys.stdout.flush()

def err(s):
  sys.stderr.write(s + '\n')
  sys.stderr.flush()

def get_query():
  query = ""
  for i in range(1, len(sys.argv)):
    if not sys.argv[i].startswith("--"):
      query += sys.argv[i] + " "
  return query.strip()

def get_param(key, default = None):
  for i in range(1, len(sys.argv)):
    if sys.argv[i].startswith("--" + key + "="):
      return sys.argv[i][len(key) + 3:]
  return default

def get_value(json, key, default = None):
  if json == None:
    return default
  return json[key] if key in json else default

def get_media(json_str):
  json = {}
  try:
    json = json.loads(json_str)
  except:
    return Media(None, Metadata({}))
  if "tv" in json:
    return TV(json["tv"], Metadata(json))
  if "movie" in json:
    return Movie(json["movie"], Metadata(json))
  return Media(None, Metadata(json))

def has_array(cmp_array, array):
  size = 0
  if array == None:
    return True
  if cmp_array == None:
    return False
  for val in array:
    if val in cmp_array:
      size += 1
  return len(array) == size

def as_int(val):
  if val == None:
    return None
  try:
    return int(val)
  except:
    return None

def load_async_torrents(torrents, index, count):
  threads = []
  size = len(torrents)
  if index + count > size:
    count = size - index
  for i in range(0, count):
    thread = threading.Thread(target = ZooqleTorrent.load_media, args = (torrents[index + i],))
    thread.daemon = True
    thread.start()
    threads.append(thread)
  for thread in threads:
    thread.join()

class Media:
  CATEGORY = "all"
  def __init__(self, json, metadata):
    self.infos = json
    self.metadata = metadata
    self.name = None
    self.year = None

  def __str__(self):
    return self.name + " - " + str(self.year)

class TV(Media):
  CATEGORY = "TV"
  def __init__(self, json, metadata):
    Media.__init__(self, json, metadata)
    self.name = get_value(json, "episode_name")
    self.season = get_value(json, "season")
    self.episode = get_value(json, "episode")
    self.year = get_value(json, "show_started")

  def __str__(self):
    return self.name + " - S" + str(self.season) + "E" + str(self.episode)

class Movie(Media):
  CATEGORY = "Movies"
  def __init__(self, json, metadata):
    Media.__init__(self, json, metadata)
    self.name = get_value(json, "name")
    self.year = get_value(json, "year")

class Metadata:
  QUALITIES = ["Low", "Med", "Std", "720p", "1080p", "Ultra"]
  def __init__(self, json):
    self.infos = get_value(json, "media_info")
    self.quality = get_value(json, "video_quality")
    self.audios = get_value(self.infos, "audio_lang")
    self.subtitles = get_value(self.infos, "subtitle_lang")
    self.size = get_value(json, "size", 0)

  def __str__(self):
    return self.quality

class ZooqleTorrent:
  NAMESPACE = "https://zooqle.com/xmlns/0.1/index.xmlns"
  def __init__(self, xml_node):
    self.title = None
    self.html_link = None
    self.torrent_link = None
    self.seeders = None
    self.leechers = None
    self.info_hash = None
    self.media = None
    self.__load__(xml_node)

  def load_media(self):
    if self.media != None:
      return self.media
    response = requests.get("https://zooqle.com/api/media/" + self.info_hash)
    self.media = get_media(response.content)
    if response.status_code != 200:
      err("load_media(" + str(self.info_hash) + "): " + str(response.status_code) + " - " + str(response.content))
    return self.media

  def __load__(self, xml_node):
    self.title = xml_node.find("title").text.encode("ascii", "ignore")
    self.html_link = xml_node.find("link").text
    self.torrent_link = xml_node.find("enclosure").attrib["url"]
    self.seeders = int(xml_node.find("{" + ZooqleTorrent.NAMESPACE + "}seeds").text)
    self.leechers = int(xml_node.find("{" + ZooqleTorrent.NAMESPACE + "}peers").text)
    self.info_hash = xml_node.find("{" + ZooqleTorrent.NAMESPACE + "}infoHash").text

  def __str__(self):
    return self.title + " [" + str(self.seeders) + ":" + str(self.leechers) + "]"

class Criteria:
  def __init__(self):
    self.query = get_query()
    self.category = get_param("category", "all")
    self.quality = get_param("quality")
    self.min_quality = get_param("min-quality")
    self.year = as_int(get_param("year"))
    self.audios = get_param("audios")
    if self.audios != None:
      self.audios = self.audios.split(',')
    self.subtitles = get_param("subtitles")
    if self.subtitles != None:
      self.subtitles = self.subtitles.split(',')
    self.season = as_int(get_param("season"))
    self.episode = as_int(get_param("episode"))
    self.seeders = as_int(get_param("seeders", 0))
    self.max_size = as_int(get_param("size"))
    self.count = as_int(get_param("count", 0))
    self.load = as_int(get_param("load", 10))

  def matches(self, torrent):
    is_matching = [
      self.__seeders__,
      self.__max_size__,
      self.__quality__,
      self.__min_quality__,
      self.__year__,
      self.__audios__,
      self.__subtitles__,
      self.__season__,
      self.__episode__
    ]
    for matches in is_matching:
      if not matches(torrent):
        return False
    return True

  def __seeders__(self, torrent):
    return torrent.seeders >= self.seeders

  def __max_size__(self, torrent):
    if self.max_size == None:
      return True
    return torrent.media.metadata.size <= self.max_size

  def __quality__(self, torrent):
    if self.quality == None:
      return True
    return torrent.media.metadata.quality == self.quality

  def __min_quality__(self, torrent):
    if self.min_quality == None:
      return True
    try:
      i = Metadata.QUALITIES.index(self.min_quality)
      j = Metadata.QUALITIES.index(torrent.media.metadata.quality)
      return j >= i
    except:
      return False

  def __year__(self, torrent):
    if self.year == None:
      return True
    return torrent.media.year == self.year

  def __audios__(self, torrent):
    return has_array(torrent.media.metadata.audios, self.audios)

  def __subtitles__(self, torrent):
    return has_array(torrent.media.metadata.subtitles, self.subtitles)

  def __season__(self, torrent):
    if self.season == None:
      return True
    if not isinstance(torrent.media, TV):
      return False
    return torrent.media.season == self.season

  def __episode__(self, torrent):
    if self.episode == None:
      return True
    if not isinstance(torrent.media, TV):
      return False
    return torrent.media.episode == self.episode

def zooqle_search(what, category = "all"):
  page = 1
  torrents = []
  has_torrents = True
  while has_torrents:
    response = requests.get("https://zooqle.com/search?q=" + urllib.quote_plus(what) + "+category%3A" + category + "&s=ns&v=t&sd=d&fmt=rss&pg=" + str(page))
    channel = ET.fromstring(response.content)[0]
    has_torrents = False
    for child in channel:
      if child.tag == "item":
        has_torrents = True
        torrents.append(ZooqleTorrent(child))
    page += 1
  return torrents

def usage():
  out("Usage: " + sys.argv[0] + " query [--category=<category>] [--quality=<quality>] [--min-quality=<quality>] [--year=<year>] [--audios=<audios>] [--subtitles=<subtitles>] [--season=<season>] [--episode=<episode>] [--seeders=<seeders>] [--size=<size>] [--count=<count>] [--load=<load>]")
  out("\tcategory\t'TV', 'Movies'")
  out("\tquality\t\t'Low', 'Med', 'Std', '720p', '1080p', 'Ultra'")
  out("\tmin-quality\tMinimum quality required")
  out("\tyear\t\tRelease year")
  out("\taudios\t\tHas audios as <audio1>,<audio2>,...")
  out("\tsubtitles\tHas subtitles as <sub1>,<sub2>,...")
  out("\tseason\t\tSeason number. Only applies for TV")
  out("\tepisode\t\tEpisode number. Only applies for TV")
  out("\tseeders\t\tMinimum number of seeders")
  out("\tsize\t\tMaximal size in bytes")
  out("\tcount\t\tNumber of results")
  out("\tload\t\tNumber of torrents loaded per iteration")
  out("")
  out("Example: " + sys.argv[0] + " rick and morty --category=TV --min-quality=720p --subtitles=en --season=3 --episode=1 --size=1073741824")

if len(sys.argv) == 1:
  usage()
  sys.exit(1)

count = 0
criteria = Criteria()
torrents = zooqle_search(criteria.query, criteria.category)
for i in range(0, len(torrents)):
  if i % criteria.load == 0:
    load_async_torrents(torrents, i, criteria.load)
  if criteria.matches(torrents[i]):
    out(str(torrents[i]) + "\t" + torrents[i].html_link + "\t" + torrents[i].torrent_link)
    count += 1
    if criteria.count != 0 and count >= criteria.count:
      break


import base64
import io
import json
import os
import time
from StringIO import StringIO

import PIL
from PIL import Image
from bson import ObjectId
from cognitive_face import CognitiveFaceException
from flask import Flask, make_response, send_file
from flask import render_template, jsonify, request, flash, redirect, Response, url_for
from flask_mongo_sessions import MongoDBSessionInterface
from flask_mongoengine.wtf import model_form
from mongoengine import DoesNotExist
from pymongo.database import Database
from werkzeug.datastructures import FileStorage
from flask_mongoengine import MongoEngine

from model.person import Person
from model.notification import Notification
from model.face import Face, FaceDocument
from model.personlegacy import PersonLegacy
from person_group import PersonGroup
from utils import get_db, JSONEncoder, initialize_cf, IGNORE_PERSON_GROUP, UNKNOWN_PERSON_GROUP, KNOWN_PERSON_GROUP, \
    setup_mongoengine, get_secret
from forms import FaceUploadForm
from notifier_rest import NotifierREST

APP = Flask(__name__)

# This is for XHR
APP.config['SECRET_KEY'] = os.environ['AZURE_CLIENT_SECRET']
# This is bad :(
APP.config['WTF_CSRF_ENABLED'] = False

setup_mongoengine(APP)
MongoEngine(APP)

db = get_db()  # type: Database
APP.session_interface = MongoDBSessionInterface(APP, db, 'sessions')  # Setup session handler using mongo

# TODO: Deprecated
test_collection = db.get_collection('test')
# TODO: Deprecated
new_faces = db.get_collection('new_faces')

face_collection = db.get_collection('faces')  # Images dump
persons_collection = db.get_collection('persons')  # Only known people

# Initialize Cognitive face
try:
    initialize_cf()
except Exception as ex:
    flash('Failed to initialize CF: {}'.format(ex), category='danger')

# use the modified encoder class to handle ObjectId & datetime object while jsonifying the response.
APP.json_encoder = JSONEncoder

@APP.errorhandler(Exception)
def exception_handler(ex):
    return jsonify({'error': ex.message}), 500

@APP.route('/')
def main_page():
    return redirect(url_for('show_all_persons'))

def show_all():
    pending_users = []
    for new_face in new_faces.find():
        object_id = ObjectId(new_face['_id'])
        img_url = url_for('get_face_image', object_id=object_id)

        train_known_face = url_for('train_face_known', object_id=object_id)
        train_ignore_face = url_for('train_face_ignore', object_id=object_id)
        delete_face_url = url_for('delete_face', object_id=object_id)
        person_info_url = url_for('person_info', object_id=object_id)

        status = 'untrained' if new_face['status'] == 'new' else new_face['status']

        pending_users.append(
            {'ts': object_id.generation_time, 'id': object_id, 'img_url': img_url, 'train_known': train_known_face,
             'train_ignore': train_ignore_face, 'delete_face_url': delete_face_url, 'info_url': person_info_url,
             'status': status})

    return render_template('pending_users.html', pending_users=pending_users)

@APP.route('/person/json/<object_id>')
def person_info(object_id):
    person = PersonLegacy.fetch(new_faces, object_id)
    return jsonify(person.document)

@APP.route('/person/identify/<object_id>')
def person_identify(object_id):
    person = PersonLegacy.fetch(new_faces, object_id)
    possible_matches = []
    for pg in [PersonGroup.known_person_group(), PersonGroup.unknown_person_group(), PersonGroup.ignore_person_group()]:
        identification_result = pg.identify(person)
        if not identification_result:
            continue

        for candidates in identification_result.values():
            for candidate in candidates:
                person_id = candidate['personId']
                matching_person = PersonLegacy.fetch_by_person_id(new_faces, person_id)
                if not matching_person:
                    continue

                possible_matches.append(matching_person.id)

        time.sleep(1)

    if person.id not in possible_matches:
        possible_matches.append(person.id)
    return jsonify(result=possible_matches), 200

@APP.route('/reset_training')
def reset_training():
    PersonGroup.reset_all()
    for face_document in new_faces.find():
        person = PersonLegacy(new_faces, face_document)
        person.remove_all_trained_details()

    flash('Reset all training group', category='info')
    return redirect('/')

@APP.route('/face/known/<object_id>')
def is_face_known(object_id):
    person = PersonLegacy.fetch(new_faces, object_id)

    person_group_id = person.get_group_person_id(KNOWN_PERSON_GROUP)
    result = person_group_id is not None

    if not result:
        result = person in PersonGroup.known_person_group()

    return jsonify({'result': result})

@APP.route('/face/unknown/<object_id>')
def is_face_unknown(object_id):
    person = PersonLegacy.fetch(new_faces, object_id)
    return jsonify({'result': person.get_group_person_id(UNKNOWN_PERSON_GROUP)})

@APP.route('/face/ignored/<object_id>')
def is_face_ignored(object_id):
    person = PersonLegacy.fetch(new_faces, object_id)
    return jsonify({'result': person.get_group_person_id(IGNORE_PERSON_GROUP)})

@APP.route('/face/image/<object_id>')
def get_face_image(object_id):
    person = PersonLegacy.fetch(new_faces, object_id)
    return Response(person.image, mimetype='image/jpeg')

@APP.route('/person/delete/<object_id>')
def delete_face(object_id):  # TODO: Sanitize this input
    person = PersonLegacy.fetch(new_faces, object_id)
    result = person.delete()
    if result is None:
        flash('Failed deleting: {}'.format(json.dumps(result)))

    return redirect('/')

@APP.route('/train_face/ignore/<object_id>')
def train_face_ignore(object_id):
    person_group = PersonGroup.ignore_person_group()
    return train_face(object_id, person_group)

@APP.route('/train_face/known/<object_id>')
def train_face_known(object_id):
    person_group = PersonGroup.known_person_group()
    return train_face(object_id, person_group)

@APP.route('/train_face/unknown/<object_id>')
def train_face_unknown(object_id):
    person_group = PersonGroup.unknown_person_group()
    return train_face(object_id, person_group)

def train_face(object_id, person_group):
    person = PersonLegacy.fetch(new_faces, object_id)
    try:
        person_group.add_person(person)
    except CognitiveFaceException as ex:
        if ex.status_code in [400, 404]:
            flash("Face not detected, can't add to group - " + person_group.name, category='warning')
            return redirect('/')
    except Exception as ex:
        flash("API Failed - {}".format(ex.message), category="danger")
        return redirect("/")

    unknown_person_group = PersonGroup.unknown_person_group()
    if not unknown_person_group.remove_person(person):
        flash("Face doesn't exist in " + unknown_person_group.name + " so it wasn't removed", category='info')
    return redirect("/")

@APP.route('/test')
def test_endpoint():
    try:
        output = []
        for s in new_faces.find():
            output.append(ObjectId(s['_id']))

        return jsonify({'result': output}), 200
    except Exception as ex:
        return ex.message, 500

def _store_uploaded_face():
    if 'faceImage' not in request.files:
        flash('No file uploaded')
        return render_template('add_face.html')

    file = request.files['faceImage']  # type: FileStorage
    if file.filename == '':
        flash('No file selected', category='danger')
        return render_template('add_face.html')

    with file.stream as f:
        b64_file = base64.b64encode(f.read())

    if b64_file is None:
        flash('Failed converting file to b64')
        return render_template('add_face.html')

    try:
        result = new_faces.insert({'image': b64_file, 'status': 'new'})
    except Exception as ex:
        flash(ex.message)
        return render_template('add_face.html')

    return redirect('/')

@APP.route('/add_face', methods=['GET', 'POST'])
def add_face():
    if request.method == 'POST':
        return _store_uploaded_face()
    else:
        return render_template('add_face.html')

def set_person_id(template, id):
    response = make_response(template)
    if id:
        response.headers['X-Person-ID'] = id

    return response

@APP.route('/face/create', methods=['POST', 'GET'])
def face_store():
    id = request.args.get('id', None)
    id_param = '?id=' + id if id else ''

    fuf = FaceUploadForm()

    response_back_to_face_create = make_response(
        render_template('face_add_form.html', form=fuf, person_id_param=id_param))
    if id:
        response_back_to_face_create.headers['X-Person-ID'] = id

    if fuf.is_submitted():
        return handle_face_upload(fuf, id, response_back_to_face_create)

    return response_back_to_face_create

def handle_face_upload(fuf, id, response_back_to_face_create):
    face = FaceDocument(image=fuf.file.data.read())
    flash('Added face id ' + str(id), category='info')

    try:
        person = Person.objects.get(id=id) if id else None
    except DoesNotExist as ex:
        person = None

    if not person:
        flash("Can't create face without person context '{}'".format(id), category='warning')
        return redirect(url_for('create_person'))

    face.person = person.id
    known_group = PersonGroup.known_person_group()

    try:
        result = known_group.add_face_to_person(person.person_id, face)
    except CognitiveFaceException as ex:
        flash(ex.msg, category='danger')
        return response_back_to_face_create

    face.save()
    person.trained_faces = person.trained_faces + [str(face.id)]  # Lists are immutable
    person.save()
    known_group.train()  # This shouldn't happen all the time but whatever
    return redirect(url_for('get_person', object_id=str(person.id)))

# @APP.route('/face_migrate')
# def face_document_migrate():
#     iter = face_collection.find(projection={'image': False})
#     for document in iter:
#         old_face = Face.find(face_collection, document['_id'])
#         new_face = FaceDocument(id=old_face.id, image=old_face.read(), person=old_face.person)
#         new_face.save()
#
#     return 'Ok', 200

@APP.route('/face/<object_id>')
def face_get(object_id):
    face = FaceDocument.objects.get(id=object_id)

    width = request.args.get('w', default=None)
    width = int(width) if width else None
    height = request.args.get('h', default=None)
    height = int(height) if height else None

    if not width and not height:
        return Response(face.image.read(), mimetype='image/jpeg')

    img = Image.open(face.image)

    if width:
        wpercent = (width / float(img.size[0]))
        height = int((float(img.size[1]) * float(wpercent)))
    elif height:
        hpercent = (height / float(img.size[1]))
        width = int((float(img.size[0]) * float(hpercent)))
    else:
        return "Error", 500

    roiImg = img.resize((width, height), PIL.Image.ANTIALIAS)  # we should probably cache this
    img_io = StringIO()
    roiImg.save(img_io, 'JPEG', quality=80)
    img_io.seek(0)
    return send_file(img_io, mimetype='image/jpeg')

@APP.route('/face/delete/<object_id>')
def face_delete(object_id):
    # Face.delete(face_collection, ObjectId(object_id))
    flash('Deleted face ' + str(object_id) + " didn't really happen")
    return redirect(url_for('show_all_faces'))

@APP.route('/face/update/<object_id>')
def face_update(object_id):
    pass  # http://wtforms.simplecodes.com/docs/0.6.1/fields.html (look at "Select fields with dynamic choice value")

@APP.route('/face')
def show_all_faces():
    faces = Face.find_all(face_collection)
    return render_template('show_all_faces.html', faces=faces)

@APP.route('/person/')
def show_all_persons():
    all_persons = Person.objects()
    return render_template('show_all_persons.html', persons=all_persons)

@APP.route('/person/<object_id>')
def get_person(object_id):
    person = Person.objects.get(id=object_id)
    return render_template('show_person.html', person=person)

@APP.route('/person/create', methods=['GET', 'POST'])
def create_person():
    PersonForm = model_form(Person)

    form = PersonForm(request.form)
    if request.method == 'POST' and form.validate():
        # Create person for PersonGroup
        person_name = form.name.data
        try:
            person = Person.objects.get(name=person_name)
        except DoesNotExist as ex:
            person = None

        if person:
            flash('Person {} already exists, not creating a new one'.format(person_name), category='info')
        else:
            person_id = PersonGroup.known_person_group().add_person_by_name(person_name)
            form.person_id.data = person_id
            form.person_group = PersonGroup.known_person_group().name
            person = form.save(validate=False)  # type: Person
            flash('Person {} successfully save'.format(person_name), category='info')

        icum_person_id = str(person.id)
        response = redirect(url_for('face_store') + '?id=' + icum_person_id)
        response.headers['X-Person-ID'] = icum_person_id

        return response

    return render_template('person_form.html', form=form)

@APP.route('/person/update/<object_id>')
def update_person(object_id):
    pass

@APP.route('/person/delete/<object_id>')
def delete_person(object_id):
    pass

@APP.route('/notifications/<notification_id>')
def show_notification(notification_id):  # Show in mobile page
    notification = Notification.objects.get(id=notification_id)
    time = notification.id.generation_time
    time_str = "{}-{}-{}, {}:{}".format(time.day, time.month, time.year, str(time.hour).zfill(2), str(time.minute).zfill(2))
    face_id = notification.icum_face_id
    face = FaceDocument.objects.get(id=face_id)
    # person = Person.objects.get(id=face.person) if face.person else None
    return render_template('show_notification.html', notification=notification, face=face, person=face.person, time=time_str)
    # return "we recognized someone at your door!\n details:{}".format(pretty)

@APP.route('/notifications/<notification_id>/dismiss', methods=['POST'])
def dismiss(notification_id):
    pass

@APP.route('/notifications/<notification_id>/add', methods=['POST'])
def new_person_from_notification(notification_id):  # Create a new person from notification image
    pass

# Dummy method to create notification online
@APP.route('/notifications/create', methods=['GET', 'POST'])
def create_notification():  # Create a new person from notification image
    NotificationForm = model_form(Notification, field_args={'msg': {'textarea': True}})

    form = NotificationForm(request.form)
    if request.method == 'POST' and form.validate():
        form.save(validate=False)
        flash('success!', category='success')

    return render_template('add_notification.html', form=form)

@APP.route('/notifications/')
def show_all_notifications():
    all_notifications = Notification.objects()
    return render_template('show_all_notifications.html', notifications=all_notifications)

def _handle_face_notification(icum_face_id, person=None, attributes=None):
    """
    No person implies that the person is unknown
    :type icum_face_id: str
    :type person: Person
    """
    if not person:
        msg = "Unknown person detected !"
        msg_type = 'unknown'
        attributes = attributes
    else:
        msg = "Person '{}' detected !".format(person.name)
        msg_type = 'known'

    notification = Notification(msg=msg, icum_face_id=icum_face_id, msg_type=msg_type, attributes=attributes)
    notification.save()

    event_hub_connection_string = get_secret("NOTIFICATION-HUB-CONNECTION-STRING")
    notifier = NotifierREST(event_hub_connection_string)
    return notifier.notify(notification)

@APP.route('/detect', methods=['POST'])
def detect():
    face_bytes = request.get_data()
    face = FaceDocument(image=face_bytes)
    face.save()

    knowns = PersonGroup.known_person_group()
    detected_face = knowns.detected_face(face.image)  # TODO: Fix for multiple people
    detected_face_guid = detected_face['id']

    identified_dict = knowns.identify_face(detected_face_guid)

    candidates = identified_dict[detected_face_guid]

    most_suitable = max(candidates, key=lambda record: record['confidence']) if candidates else None
    person = Person.objects.get(person_id=most_suitable['personId']) if candidates else None
    face.person = person.id if person else None
    face.save()
    icum_face_id = str(face.id)

    if not person:
        _handle_face_notification(icum_face_id, attributes=detected_face['attributes'])
        return jsonify(status="person is unknown"), 404

    msg = _handle_face_notification(icum_face_id, person)
    return jsonify(candidates=candidates, status='Notified person icum id {}'.format(str(person.id))), msg.status

if __name__ == '__main__':
    APP.run(debug=True)


from fastapi import APIRouter, Depends, status
from sqlalchemy.orm import Session

from ..database.setup import get_db
from ..database import models, schemas

from ..authentication import oauth2

# load business logic
from ..controller import ctr_sock

import os

# build routes
router = APIRouter(
    prefix=os.environ.get("API_URL", "/fastapi/v1") + "/user/sock", tags=["Socks"]
)

@router.get(
    "s/",
    response_model=list[schemas.ShowSock],
    dependencies=[Depends(oauth2.check_active)],
    status_code=200,
)
async def get_all_socks(
    db: Session = Depends(get_db),
    current_user: schemas.ShowUser = Depends(oauth2.get_current_user),
):
    return ctr_sock.show_all(current_user.username, db)

@router.get(
    "/{id}",
    response_model=schemas.ShowSock,
    dependencies=[Depends(oauth2.check_active)],
    status_code=200,
)
async def get_sock(
    id: int,
    db: Session = Depends(get_db),
    current_user: schemas.ShowUser = Depends(oauth2.get_current_user),
):
    return ctr_sock.show_specific(current_user.username, id, db)

@router.post("/", response_model=schemas.ShowSock, status_code=201)
async def create_sock(
    request: schemas.CreateUpdateSock,
    db: Session = Depends(get_db),
    current_user: schemas.ShowUser = Depends(oauth2.get_current_user),
):
    return ctr_sock.create_sock(current_user.username, request, db)

@router.put(
    "/{id}",
    response_model=schemas.ShowSock,
    status_code=202,
    dependencies=[Depends(oauth2.check_active)],
)
async def edit_sock(
    id: int,
    request: schemas.CreateUpdateSock,
    db: Session = Depends(get_db),
    current_user: schemas.ShowUser = Depends(oauth2.get_current_user),
):
    return ctr_sock.edit_sock(current_user.username, id, request, db)

# permission handling superuser!
@router.delete(
    "/{id}", status_code=204, dependencies=[Depends(oauth2.check_active)]
)  # , response_model=schemas.SimplyUser)
async def delete_sock(
    id: int,
    db: Session = Depends(get_db),
    current_user: schemas.ShowUser = Depends(oauth2.get_current_user),
):
    # as long as response is set to 204
    # we do net get a JSON as return :-( !
    return ctr_sock.delete_sock(current_user.username, id, db)


import multiprocessing
import multiprocessing.pool
import traceback
import time
import gc

from . names import *
from . derived import *
from . cloud import *
from . common import timed

def addFields(df, name_dict):
    for new, orig in name_dict.items():
        df[new] = df[orig]

def normalizeFields(p, show_fields=False, allow_missing=False):
    names = p.columns.tolist()
    # Ensure the alphanumerical model IDs are interpreted as strings
    p.Model = p.Model.astype('str')

    toner = findFields(names, '.*Toner.Bottle.%s.Remaining.Toner.(?!previous).*','Toner.%s', allow_missing=allow_missing)
    addFields(p, toner)

    try:
        toner_bw =  findFields(names, '.*Toner.status.Percentage','Toner.K', allow_missing=allow_missing,colors=False)

        addFields(p,toner_bw)
        toner.update(toner_bw)
    except:
        pass

    toner_names = list(toner.keys())

    toner_end = findFields(names, '.*Toner.Bottle.%s.End.SP7.*', 'Toner.End.Status.%s', allow_missing=allow_missing)
    addFields(p, toner_end)

    toner_end_date = findFields(names, '.*Toner.Bottle.%s.End.Date.*', 'Toner.End.Date.%s', allow_missing=allow_missing)
    addFields(p, toner_end_date)

    pages = findFields(names, '.*Pages.Current.Toner.%s.(?!previous).*', 'Pages.Toner.%s', allow_missing=allow_missing)
    addFields(p, pages)

    prev_pages = findFields(names, '.*Pages.Current.Toner.%s.previous.*', 'Pages.Previous.Toner.%s', allow_missing=allow_missing)
    addFields(p, prev_pages)

    cov = findFields(names, '.*Pixel.Coverage.Accumulation.Coverage.%s.*', 'Coverage.%s', allow_missing=allow_missing)
    addFields(p, cov)
    cov_names = list(cov.keys())

    dev_replacement = findFields(
        names,
        ['.*Replacement.Date.Developer.%s.*',
         '.*Unit.Replacement.Date.Dev.Unit.%s.*'],
         'Developer.Replacement.Date.Recorded.%s',
         allow_missing=True,
    )
    addFields(p, dev_replacement)
    # Convert 'yymmdd' representation of dev unit replacement dates to datetime
    for f in dev_replacement.keys():
        p[f] = pd.to_datetime(p[f], format='%y%m%d', errors='coerce')

    pcus = findFields(names, '.*Estimated.Usage.Rate.PCU.%s', 'PCU.Yield.%s', allow_missing=allow_missing)
    addFields(p, pcus)
    pcus_names = list(pcus.keys())

    total_pages = findFields(names, '.*Total.Total.PrtPGS.SP8.*', 'Pages', colors=False, allow_missing=allow_missing)
    addFields(p, total_pages)
    pages_names = list(total_pages.keys())

    copy_bw_pages = findFields(names, '.*Total.PrtPGS.Color.Mode.B.W.SP8.*', 'Pages.Copy.BW', colors=False, allow_missing=allow_missing)
    addFields(p, copy_bw_pages)
    pages_names += list(copy_bw_pages.keys())

    print_bw_pages = findFields(names, '.*Total.PrtPGS.Print.Color.Mode.B.W.SP8.*', 'Pages.Print.BW', colors=False, allow_missing=allow_missing)
    addFields(p, print_bw_pages)
    pages_names += list(print_bw_pages.keys())

    # Include jobs with pages as same requirements and semantically similar
    total_jobs = findFields(names, '.*Total.Total.Jobs.SP8.*', 'Jobs', colors=False, allow_missing=allow_missing)
    addFields(p, total_jobs)
    pages_names = list(total_jobs.keys())

    used_bottles = findFields(names, ['.*used.cartridge.Total.%s',
                                      '.*Toner.Use.Count.%s'],'Toner.Bottles.Total.%s', allow_missing=allow_missing)
    addFields(p, used_bottles)
    used_bottles_names = list(used_bottles.keys())

    dev_rotation = findFields(
        names,
        ['.*(?<!Previous.Unit).PM.Counter.Rotation.Developer.%s.*',
         '.*Display.Distance.Dev.Unit.%s.SP7.942',
         '.*Drive.Distance.Counter.%s_Developer.*',
        ],
        'Developer.Rotation.%s',
        take_first=True,
        allow_missing=allow_missing,
    )
    addFields(p, dev_rotation)

    try:
        bw_dev_rotation= findFields(names, '.*Drive.Distance.Counter.Development.*','Developer.Rotation.K',allow_missing=allow_missing,colors=False)

        addFields(p,bw_dev_rotation)
        dev_rotation.update(bw_dev_rotation)
    except:
        pass

    dev_rotation_names = list(dev_rotation.keys())

    # Always allow missing for toner call threshold as this comes in mutually exclusive variants
    toner_call_threshold = findFields(names, '.*Toner.Call.Threshold.SP5.*', 'Toner.Call.Threshold', colors=False, allow_missing=True)
    addFields(p, toner_call_threshold)
    toner_call_threshold_colored = findFields(names, '.*Toner.Call.Threshold.%s.*', 'Toner.Call.Threshold.%s', colors=['K', 'CMY'], allow_missing=True)
    addFields(p, toner_call_threshold_colored)
    toner_call_timing = findFields(names, '.*Toner.Call.Timing.*', 'Toner.Call.Timing', colors=False, allow_missing=allow_missing)
    addFields(p, toner_call_timing)

    if show_fields:
        print("Toner level fields:")
        print(toner)
        print("Toner end status:")
        print(toner_end)
        print("Toner end date:")
        print(toner_end_date)
        print("Current toner pages:")
        print(pages)
        print("Previous toner pages:")
        print(prev_pages)
        print("Coverage:")
        print(cov)
        print("Dev unit replacement:")
        print(dev_replacement)
        print("Dev unit rotations:")
        print(dev_rotation)
        print("Total Pages:")
        print(total_pages)
        print("Used Bottles:")
        print(used_bottles)
        print("Dev unit rotations:")
        print(dev_rotation)
        print("PCUs:")
        print(pcus)

    return toner_names, cov_names, pages_names, used_bottles_names, dev_rotation_names, pcus_names

# Use global variables for shared data as workaround for limitations of multiprocessing module
# Note that this means that processFile() is not thread-safe.
process_df = None
def applyColorSetInner(args):
    f, color = args
    return f(process_df, color)

def applyColorSet(f, colors):
    with multiprocessing.Pool(len(colors)) as pool:
        res_parts = pool.map(applyColorSetInner, [(f, color) for color in colors], chunksize=1)
    res = pd.DataFrame()
    for part in res_parts:
        #print(part)
        duplicated = set(res.columns).intersection(part.columns)
        if duplicated:
            raise Exception(f"Duplicated columns: {', '.join(duplicated)}")
        res = pd.concat([res, part], axis=1, sort=False)
    return res

def processFile(s3_url, show_fields=False, keep_orig=False, allow_missing=False, toner_stats=True, nz_only=False, in_bucket=None, skip_processing=False, require_color_match=True):
    global process_df

    start_time = time.time()

    # This is memory intensive, so garbage collect to free up any lingering allocations
    gc.collect()

    def status(text):
        now = time.time()
        delta_s = now - start_time
        print(f"{text} [{delta_s:.2f}s elapsed for {s3_url}]")

    status("Reading data from S3")
    p = readFromS3(s3_url)

    # At time of writing the RNZ Falcon files contain data for both AU and NZ machines. This is a workaround to restrict the result to NZ machines where required.
    # TODO: implement by checking if serial is in the regional MRP file rather than using exported NZ DB content (this should be general as well as more current)
    if nz_only:
        p = p[p.Serial.isin(process_nz_sers)]

    if skip_processing:
        return p

    orig_fields = p.columns

    status("Normalizing field names")
    toner_names, cov_names, pages_names, used_bottles_names, dev_rotation_names, pcus_names = normalizeFields(p, show_fields, allow_missing)
    colors_matched = [c for c in colors_norm if f'Toner.{c}' in toner_names]
    print(colors_matched)
    if require_color_match and not colors_matched:
        raise Exception(f"No colors matched for {s3_url}")

    if not keep_orig:
        print("Dropping original fields")
        to_use = set(p.columns).difference(orig_fields)
        # Keep these regardless
        to_use.update([
            'Serial',
            'Model',
            'RetrievedDate',
            'RetrievedDateTime',
            'FileDate',
        ])
        p = p[to_use]

    # Derive total BW and color page counts
    print("Adding page count stats")
    p['Pages.BW'] = p['Pages.Print.BW'] + p['Pages.Copy.BW']
    p['Pages.Color'] = p['Pages'] - p['Pages.BW']
    pages_names += ['Pages.BW', 'Pages.Color']

    # Add delta and replacements
    times = ['RetrievedDate', 'RetrievedDateTime']
    to_add = list(toner_names + cov_names + times + pages_names + used_bottles_names + dev_rotation_names + pcus_names)
    time_deltas = [x+'.delta' for x in times]

    status(f"Adding deltas, rates and replacement fields for {s3_url}")
    addDeltas(p, to_add)
    # Coerce values to timedeltas / NaT
    for d in time_deltas:
        p[d] = pd.to_timedelta(p[d])
    p['RetrievedDateTime.delta.days']=p['RetrievedDateTime.delta'] / datetime.timedelta(days=1)
    addRates(p, to_add)
    addReplacements(p, toner_names)
    # Toner counts down passage of time, PCUs count up - so a negative delta indicates a replacement
    addReplacements(p, pcus_names, negative=True)

    process_df = sortReadings(p)

    if colors_matched and toner_stats:
        def apply(f):
            global process_df
            res = applyColorSet(f, colors_matched)
            process_df = pd.concat([process_df, res], axis=1, sort=False)

        status("Adding toner bottle indices")
        apply(indexToner)
        if pcus_names:
            status("Adding PCU indices")
            apply(indexPCU)

        status("Adding final page count for current toner")
        apply(getTonerPages)

        status("Deriving dev unit replacement dates")
        apply(deriveDevReplacements)
        apply(selectDevReplacementDate)

        status("Project pages for toner bottles and calculate ratios vs. actual pages")
        apply(projectPages)

        status("Project coverage for toner bottles and calculate ratios vs. estimated coverage")
        apply(projectCoverage)

        status("Add toner usage ratios")
        apply(getTonerRatio)

    res = process_df
    process_df = None
    status("Finished")
    return res

def summarizeTonerStats(p):
    print(f"Generating summary for {s3_url}")
    x=p[['Serial','RetrievedDateTime.delta.days']+replacement_fields].groupby('Serial')
    summary=x.sum()

    addSumRate(summary, replacement_fields)
    replacement_rates = [name+".rate" for name in replacement_fields]
    return p

def sortReadings(df):
    df.sort_values(['Serial', 'RetrievedDateTime'], inplace=True, ascending=False)
    return df

def indexToner(df, color='K'):
    prev = df.shift(-1)
    new_ser = df.Serial != prev.Serial
    new = new_ser | df[f'Toner.{color}.replaced']
    new_labels = df.index
    f = f'TonerIndex.{color}'
    res = new_labels.to_frame(name=f)
    res[f] = new_labels.where(new)
    res[f] = res[f].fillna(method='bfill')
    return res

def indexPCU(df, color='K'):
    prev = df.shift(-1)
    new_ser = df.Serial != prev.Serial
    new = new_ser | df[f'PCU.Yield.{color}.replaced']
    new_labels = df.index
    f = f'PCUIndex.{color}'
    res = new_labels.to_frame(name=f)
    res[f] = new_labels.where(new)
    res[f] = res[f].fillna(method='bfill')
    return res

def deriveDevReplacements(df, color='K'):
    rotation = f'Developer.Rotation.{color}'
    prev = df.shift(-1)
    new = df[rotation] < prev[rotation]
    new[-1] = True
    dates = pd.to_datetime(df.RetrievedDate)
    f = f'Developer.Replacement.Date.Derived.{color}'
    res = dates.to_frame(name=f)
    res[f] = dates.where(new)
    res[f] = res[f].fillna(method='bfill')
    return res

def selectDevReplacementDate(df, color='K'):
    priorities = [
        f'Developer.Replacement.Date.Recorded.{color}',
        f'Developer.Replacement.Date.Derived.{color}',
    ]
    for f in priorities:
        if f in df.columns:
            df[f'Developer.Replacement.Date.{color}'] = df[f]
            break
    return df[[f'Developer.Replacement.Date.{color}']]

# Take only rows with summary stats for toner bottles
def selectTonerStats(df):
    colnames = df.columns[df.columns.str.contains('^Projected.Pages.*')]
    cols = df[colnames].dropna(how='all')
    res = df.loc[cols.index]
    return res

def doProcessing(args, summary_rows_only=False):
    path, kwargs = args
    try:
        bucket_name = kwargs.get('in_bucket', in_bucket_name)
        res = processFile(f"s3://{bucket_name}/{path}", **kwargs)
        if summary_rows_only:
            res = selectTonerStats(res)
    except:
        print(f"Exception processing {path}")
        traceback.print_exc()
        res=None
    return res

# https://stackoverflow.com/questions/6974695/python-process-pool-non-daemonic
class NoDaemonProcess(multiprocessing.Process):
    # make 'daemon' attribute always return False
    def _get_daemon(self):
        return False
    def _set_daemon(self, value):
        pass
    daemon = property(_get_daemon, _set_daemon)

class NonDaemonPool(multiprocessing.pool.Pool):
    Process = NoDaemonProcess

# Global to make NZ serial list available for children - TODO: replace with MRP data
process_nz_sers = None

@timed
def buildDataset(to_use, kwargs={}, num_procs=None, f=doProcessing):
    if num_procs is None:
        num_procs = int(np.ceil(multiprocessing.cpu_count() / 4))
    global process_nz_sers
    ser_df = pd.read_csv('s3://ricoh-prediction-misc/mif/current/customer.csv')
    process_nz_sers = ser_df.SerialNo

    args = [(path, kwargs) for path in to_use]
    # Use non-daemonic pool as we want children to be able to fork
    with NonDaemonPool(num_procs) as pool:
        res_parts = pool.map(doProcessing, args, chunksize=1)
    res_parts = [x for x in res_parts if x is not None]
    print("Combining result parts")
    res = pd.concat(res_parts)
    return res