# -*- coding: utf-8 -*-
"""
Created on Tue Mar  2 09:47:56 2021

@author: ebbek
"""
import pandas as pd
import numpy as np
from inflhist import inflhist
import scipy.stats as sp
from infl_concat import infl_concat
from inflhist import inflhist
from figure_formatting import figure_formatting
from AUcolor import AUcolor
import os
import matplotlib.dates as mdates
from matplotlib.ticker import ScalarFormatter
import matplotlib.pyplot as plt
from matplotlib.colors import LinearSegmentedColormap
from matplotlib import font_manager as fm
from matplotlib.gridspec import GridSpec
import warnings
warnings.filterwarnings("ignore")
colors,color_names = AUcolor()
title_font = fm.FontProperties(fname='AUfonts/AUPassata_Bold.ttf',size=22)
subtitle_font = fm.FontProperties(fname='AUfonts/AUPassata_Rg.ttf',size=14)
figure_font = fm.FontProperties(fname='AUfonts/AUPassata_Bold.ttf',size=18)
ticks_font = fm.FontProperties(fname='AUfonts/AUPassata_Rg.ttf',size=18)
country_code_font = fm.FontProperties(fname='AUfonts/AUPassata_Bold.ttf',size=18)
plt.close('all')
path_parent = os.path.dirname(os.getcwd())
gendatadir = path_parent + '/gendata/' # Directory in which general data is located
moddatadir = path_parent + '/moddata/' # Directory in which modelled inflow time series are located
resdatadir = path_parent + '/resdata/' # Directory in which calibrated modelled inflow time series are located
histdatadir = path_parent + '/histdata/' # Directory in which historical inflow time series are located
figuredir = path_parent + '/figure/' # Directory in which saved figures are located
#%% ============================ INPUT ========================================
gcm_list = ['MPI-M-MPI-ESM-LR','ICHEC-EC-EARTH','CNRM-CERFACS-CNRM-CM5','MOHC-HadGEM2-ES', 'NCC-NorESM1-M'] # General Circulation Model
rcm_list = ['RCA4','HIRHAM5'] # Regional Climate Model
rcp_list = ['85'] # Representative Concentration Pathways
hydrotype = 'HDAM' # Type of hydropower plant
WI = 1 # Wattsight data included (1) or not (0)
#%% ============================= OUTPUT ======================================
if WI == 1:
    country_name = ['Norway','France','Spain','Switzerland','Sweden','Germany','Austria','Italy',
                    'Bulgaria','Croatia','Portugal', 'Romania','Czech_Republic', 'Hungary',
                    'Bosnia_and_Herzegovina','Serbia','Slovenia','Finland','Poland','Slovakia',
                    'North_Macedonia','Montenegro']
    country_iso_alpha_2 = ['NO','FR','ES','CH','SE','DE','AT','IT','BG','HR','PT','RO',
                            'CZ','HU','BA','RS','SI','FI','PL','SK','MK','ME']
    nrows = 8
    ncols = 3
    lp = -2
else:
    country_name = ['Norway','France','Spain','Switzerland','Sweden','Austria','Italy', 'Romania',
                    'Bulgaria','Portugal', 'Montenegro', 'Serbia']
    country_iso_alpha_2 = ['NO','FR','ES','CH','SE','AT','IT','RO','BG','PT', 'ME', 'RS']
    nrows = 4
    ncols = 3
    lp = 0

# nrows = 1
# ncols = 1
# country_name = ['Norway']
# country_iso_alpha_2 = ['NO']

if nrows > 1:
    if len(country_iso_alpha_2) % 3 != 0:
        fig,ax = figure_formatting('','',1,1,figsiz = (17,14))
        gs = GridSpec(nrows, ncols)
        ax = [0]*len(country_iso_alpha_2)
        ax_twin = [0]*len(country_iso_alpha_2)
        counter = 0
        for row in range(nrows-1):
            for col in range(ncols):
                ax[counter] = plt.subplot(gs[row,col])
                ax[counter].tick_params(axis='y', colors=colors[0])
                ax[counter].set_yticklabels(ax[counter].get_yticks(), fontProperties = ticks_font)
                ax[counter].set_xticklabels(ax[counter].get_xticks(), fontProperties = ticks_font)
                ax[counter].yaxis.set_major_formatter(ScalarFormatter(useMathText=True, useOffset=False))
                ax[counter].xaxis.set_major_formatter(ScalarFormatter(useMathText=True, useOffset=False))
                ax_twin[counter] = ax[counter].twinx()
                ax_twin[counter].tick_params(axis='y', colors=colors[6])
                ax_twin[counter].set_yticklabels(ax_twin[counter].get_yticks(), fontProperties = ticks_font)
                ax_twin[counter].yaxis.set_major_formatter(ScalarFormatter(useMathText=True, useOffset=False))
                counter += 1
        if len(country_iso_alpha_2) % 3 == 1:
            ax[counter] = plt.subplot(gs[row+1,0])
            ax[counter].tick_params(axis='y', colors=colors[0])
            ax[counter].set_yticklabels(ax[counter].get_yticks(), fontProperties = ticks_font)
            ax[counter].set_xticklabels(ax[counter].get_xticks(), fontProperties = ticks_font)
            ax[counter].yaxis.set_major_formatter(ScalarFormatter(useMathText=True, useOffset=False))
            ax[counter].xaxis.set_major_formatter(ScalarFormatter(useMathText=True, useOffset=False))
            ax_twin[counter] = ax[counter].twinx()
            ax_twin[counter].tick_params(axis='y', colors=colors[6])
            ax_twin[counter].set_yticklabels(ax_twin[counter].get_yticks(), fontProperties = ticks_font)
            ax_twin[counter].yaxis.set_major_formatter(ScalarFormatter(useMathText=True, useOffset=False))
        elif len(country_iso_alpha_2) % 3 == 2:
            ax[counter] = plt.subplot(gs[row+1,0])
            ax[counter].tick_params(axis='y', colors=colors[0])
            ax[counter].set_yticklabels(ax[counter].get_yticks(), fontProperties = ticks_font)
            ax[counter].set_xticklabels(ax[counter].get_xticks(), fontProperties = ticks_font)
            ax[counter].yaxis.set_major_formatter(ScalarFormatter(useMathText=True, useOffset=False))
            ax[counter].xaxis.set_major_formatter(ScalarFormatter(useMathText=True, useOffset=False))
            ax_twin[counter] = ax[counter].twinx()
            ax_twin[counter].tick_params(axis='y', colors=colors[6])
            ax_twin[counter].set_yticklabels(ax_twin[counter].get_yticks(), fontProperties = ticks_font)
            ax_twin[counter].yaxis.set_major_formatter(ScalarFormatter(useMathText=True, useOffset=False))
            ax[counter+1] = plt.subplot(gs[row+1,1])
            ax[counter+1].tick_params(axis='y', colors=colors[0])
            ax[counter+1].set_yticklabels(ax[counter+1].get_yticks(), fontProperties = ticks_font)
            ax[counter+1].set_xticklabels(ax[counter+1].get_xticks(), fontProperties = ticks_font)
            ax[counter+1].yaxis.set_major_formatter(ScalarFormatter(useMathText=True, useOffset=False))
            ax[counter+1].xaxis.set_major_formatter(ScalarFormatter(useMathText=True, useOffset=False))
            ax_twin[counter+1] = ax[counter+1].twinx()
            ax_twin[counter+1].tick_params(axis='y', colors=colors[6])
            ax_twin[counter+1].set_yticklabels(ax_twin[counter+1].get_yticks(), fontProperties = ticks_font)
            ax_twin[counter+1].yaxis.set_major_formatter(ScalarFormatter(useMathText=True, useOffset=False))
    fig.subplots_adjust(wspace=0.3)
    fig.subplots_adjust(hspace=0.4)
    fig1,ax1 = figure_formatting('','',1,1,color='k',figsiz = (17,14))
    gs1 = GridSpec(nrows, ncols)
    ax1 = [0]*len(country_iso_alpha_2)
    counter = 0
    for row in range(nrows-1):
        for col in range(ncols):
            ax1[counter] = plt.subplot(gs1[row,col])
            ax1[counter].tick_params(axis='y', colors=colors[0])
            ax1[counter].set_yticklabels(ax1[counter].get_yticks(), fontProperties = ticks_font)
            ax1[counter].set_xticklabels(ax1[counter].get_xticks(), fontProperties = ticks_font)
            ax1[counter].yaxis.set_major_formatter(ScalarFormatter(useMathText=True, useOffset=False))
            ax1[counter].xaxis.set_major_formatter(ScalarFormatter(useMathText=True, useOffset=False))
            counter += 1
    if len(country_iso_alpha_2) % 3 == 1:
        ax1[counter] = plt.subplot(gs[row+1,0])
        ax1[counter].tick_params(axis='y', colors='k')
        ax1[counter].set_yticklabels(ax1[counter].get_yticks(), fontProperties = ticks_font)
        ax1[counter].set_xticklabels(ax1[counter].get_xticks(), fontProperties = ticks_font)
        ax1[counter].yaxis.set_major_formatter(ScalarFormatter(useMathText=True, useOffset=False))
        ax1[counter].xaxis.set_major_formatter(ScalarFormatter(useMathText=True, useOffset=False))
    elif len(country_iso_alpha_2) % 3 == 2:
        ax1[counter] = plt.subplot(gs[row+1,0])
        ax1[counter].tick_params(axis='y', colors='k')
        ax1[counter].set_yticklabels(ax1[counter].get_yticks(), fontProperties = ticks_font)
        ax1[counter].set_xticklabels(ax1[counter].get_xticks(), fontProperties = ticks_font)
        ax1[counter].yaxis.set_major_formatter(ScalarFormatter(useMathText=True, useOffset=False))
        ax1[counter].xaxis.set_major_formatter(ScalarFormatter(useMathText=True, useOffset=False))
        ax1[counter+1] = plt.subplot(gs[row+1,1])
        ax1[counter+1].tick_params(axis='y', colors='k')
        ax1[counter+1].set_yticklabels(ax1[counter+1].get_yticks(), fontProperties = ticks_font)
        ax1[counter+1].set_xticklabels(ax1[counter+1].get_xticks(), fontProperties = ticks_font)
        ax1[counter+1].yaxis.set_major_formatter(ScalarFormatter(useMathText=True, useOffset=False))
        ax1[counter+1].xaxis.set_major_formatter(ScalarFormatter(useMathText=True, useOffset=False))
    fig1.subplots_adjust(wspace=0.3)
    fig1.subplots_adjust(hspace=0.4)
else:
    fig,ax,ax_twin = figure_formatting('','',nrows,ncols,figsiz = (18,12.5),ylab_twin='',color_twin=colors[6],twin=True)
    fig1,ax1 = figure_formatting('','',nrows,ncols,color='k',figsiz = (18,12.5))
rcp = rcp_list[0]
for c in range(len(country_name)):
    gcm_it = 0
    count = 0
    country = country_iso_alpha_2[c]
    country_l = country_name[c]
    hist = inflhist(histdatadir,1991,2020,country,country_l)*1e-3
    hist = hist[(hist.T != 0).any()] # Removing rows with zeros
    hist_m_train = hist #.drop(columns='month')
    hist_m_train['month'] = hist_m_train.index.month
    hist_m_train_seasonal = hist_m_train.groupby('month').mean()
    ind = pd.date_range('2016/01/01','2016/12/31',freq='MS')
    hist_m_train_seasonal.set_index(ind,inplace=True)
    ax[c].text(0.01, 0.7,country, transform=ax[c].transAxes,fontproperties=ticks_font,zorder=10)
    ax1[c].text(0.01, 0.7,country, transform=ax1[c].transAxes,fontproperties=ticks_font,zorder=10)
    pearson = [0]*len(gcm_list)*len(rcm_list)
    if c == 0:
        ax[c].plot(hist_m_train_seasonal,label='Historical',color=colors[0],zorder=20,lw=2,linestyle='--')
        # ax[c].set_ylabel('Historical inflow [TWh]')
    else:
        ax[c].plot(hist_m_train_seasonal,color=colors[0],zorder=5,lw=2,linestyle='--')
    for gcm in gcm_list:
        rcm_it = 0
        gcm = gcm_list[gcm_it]
        for rcm in rcm_list:
            rcm = rcm_list[rcm_it]
            infl_cal,infl_EOC = infl_concat(country,gendatadir,moddatadir,gcm,rcm,rcp)
            infl_cal = (infl_cal*(3.6e12)**-1)*1e-3 # Unit conversion from Joule to TWh
            time_dt = pd.to_datetime(infl_cal.index)
            infl_cal['date'] = time_dt
            infl_cal.set_index('date',inplace=True)
            infl_cal_monthly = infl_cal.groupby(pd.Grouper(freq='MS')).sum()
            infl_cal_monthly['month'] = infl_cal_monthly.index.month
            infl_cal_m_train_seasonal = infl_cal_monthly.groupby('month').mean()
            infl_cal_m_train_seasonal.set_index(ind,inplace=True)
            # ax_twin[c].set_ylabel('Modelled inflow [TWh]')
            pearson[count] = hist_m_train_seasonal.corrwith(infl_cal_m_train_seasonal).inflow.round(2)
            RF = hist_m_train_seasonal.values/infl_cal_m_train_seasonal.values
            if c == 0:
                ax_twin[c].plot(infl_cal_m_train_seasonal,color=colors[1+count],label=gcm + '-' + rcm,lw=1)
                ax1[c].plot(infl_cal_m_train_seasonal.index,RF,marker='o', linestyle='dashed',markersize=4,color=colors[1+count],label=gcm + '-' + rcm,lw=1)
            else:
                ax_twin[c].plot(infl_cal_m_train_seasonal,color=colors[1+count],lw=1)
                ax1[c].plot(infl_cal_m_train_seasonal.index,RF,marker='o', linestyle='dashed',markersize=4,color=colors[1+count],lw=1)
            count += 1
            rcm_it += 1
        gcm_it += 1
    if np.mean(pearson) >= 0:
        ax[c].text(0.8, 0.7,'r=' + str(np.mean(pearson).round(2)), transform=ax[c].transAxes,fontproperties=ticks_font,zorder=10)
    else:
        ax[c].text(0.77, 0.7,'r=' + str(np.mean(pearson).round(2)), transform=ax[c].transAxes,fontproperties=ticks_font,zorder=10)
    ax[c].xaxis.set_major_formatter(mdates.DateFormatter("%b"))
    ax[c].set_xlim([min(ind),max(ind)])
    ax[c].xaxis.set_minor_locator(mdates.MonthLocator(interval=1))
    ax[c].xaxis.set_major_locator(mdates.MonthLocator(interval=2))
    ax1[c].xaxis.set_major_formatter(mdates.DateFormatter("%b"))
    ax1[c].set_xlim([min(ind),max(ind)])
    ax1[c].xaxis.set_minor_locator(mdates.MonthLocator(interval=1))
    ax1[c].xaxis.set_major_locator(mdates.MonthLocator(interval=2))

fig.legend(frameon=False,loc='lower center',ncol=3,prop=ticks_font,handletextpad=0.1,labelspacing=0,borderaxespad=-0.5)
fig1.legend(frameon=False,loc='lower center',ncol=3,prop=ticks_font,handletextpad=0.1,labelspacing=0,borderaxespad=-0.5)

if len(country_name) == 2:
    fig.savefig(figuredir + 'Model_evaluation_' + country_name[0] + '_' + country_name[1] + '_' + rcp + '.png',bbox_inches='tight')
    fig1.savefig(figuredir + 'Retain_factors_' + country_name[0] + '_' + country_name[1] + '_' + rcp + '.png',bbox_inches='tight')
elif len(country_name) == 22:
    fig.savefig(figuredir + 'Model_evaluation_' + rcp + '.png',bbox_inches='tight')
    fig1.savefig(figuredir + 'Retain_factors_' + rcp + '.png',bbox_inches='tight')


from flask_app import app
from flask import render_template, redirect, request, session
from flask_app.models.dojo import Dojo
from flask_app.models.ninja import Ninja

@app.route('/create_dojo', methods=['POST'])
def create_user():
    data={
        'name':request.form['name'],
    }
    Dojo.create_dojo(data)
    return redirect('/dojos')

@app.route('/dojos')
def alldojo():
    alldojo = Dojo.getAllDojos()
    return render_template ('dojos.html', dojos = alldojo)

@app.route('/delete/<int:id>')
def deleteuser(id):
    data={
        'id':id
    }
    Dojo.delete_dojo(data)
    return redirect('/dojos')


from pathlib import Path
from copy import deepcopy
from typing import Callable, Optional, Any, \
    Union, Generator, TextIO

import numpy as np

from dae.utils.variant_utils import get_interval_locus_ploidy

from dae.variants_loaders.raw.flexible_variant_loader import \
    flexible_variant_loader

from dae.variants.core import Allele
from dae.genomic_resources.reference_genome import ReferenceGenome
from dae.pedigrees.families_data import FamiliesData

def _cnv_location_to_vcf_trasformer() \
        -> Callable[[dict[str, Any]], dict[str, Any]]:
    """Expand shorthand loc notation to separate crom, pos, pos_end attrs.

    In case the input uses CNV location this transformer will produce
    internal (chrom, pos, pos_end) description of the CNV position.
    """
    def transformer(result: dict[str, Any]) -> dict[str, Any]:
        location = result["location"]
        chrom, pos_range = location.split(":")
        beg, end = pos_range.split("-")
        result["chrom"] = chrom
        result["pos"] = int(beg)
        result["pos_end"] = int(end)

        return result

    return transformer

def _cnv_vcf_to_vcf_trasformer() \
        -> Callable[[dict[str, Any]], dict[str, Any]]:
    """Convert pos and pos_end to int.

    In case the input uses VCF-like description of the CNVs this
    transformer will check it and handle the proper type conversion for
    `pos` and `pos_end` values.
    """
    def trasformer(result: dict[str, Any]) -> dict[str, Any]:
        chrom = result["chrom"]
        pos = int(result["pos"])
        pos_end = int(result["pos_end"])

        result["chrom"] = chrom
        result["pos"] = pos
        result["pos_end"] = pos_end

        return result

    return trasformer

def _configure_cnv_location(
        header: list[str],
        transformers: list[Callable[[dict[str, Any]], dict[str, Any]]],
        cnv_chrom: Optional[str] = None,
        cnv_start: Optional[str] = None,
        cnv_end: Optional[str] = None,
        cnv_location: Optional[str] = None) -> None:
    """Configure the header and position-handling transformers.

    This helper function will **configure** the header and transformers needed
    to handle position of CNVs in the input record.
    """
    if cnv_chrom is not None or cnv_start is not None or \
            cnv_end is not None:
        if cnv_location is not None:
            raise ValueError(
                f"mixed variant location definitions: "
                f"vcf({cnv_chrom}:{cnv_start}-{cnv_end}) and "
                f"location({cnv_location})")
        if cnv_chrom is None:
            cnv_chrom = "chrom"
        if cnv_start is None:
            cnv_start = "pos"
        if cnv_end is None:
            cnv_end = "pos_end"

        chrom_index = header.index(cnv_chrom)
        start_index = header.index(cnv_start)
        end_index = header.index(cnv_end)

        header[chrom_index] = "chrom"
        header[start_index] = "pos"
        header[end_index] = "pos_end"
        transformers.append(_cnv_vcf_to_vcf_trasformer())
    else:
        if cnv_location is None:
            cnv_location = "location"
        location_index = header.index(cnv_location)
        header[location_index] = "location"
        transformers.append(_cnv_location_to_vcf_trasformer())

def _cnv_dae_best_state_to_best_state(
        families: FamiliesData, genome: ReferenceGenome) \
        -> Callable[[dict[str, Any]], dict[str, Any]]:
    """Transform old dae family id/best state to canonical form.

    In case the genotype of the CNVs is specified in old
    dae family id/best state notation, this transformer will handle it
    and transform it to canonical family id/best state form
    """
    def transformer(result: dict[str, Any]) -> dict[str, Any]:
        variant_type = result["variant_type"]
        actual_ploidy = np.fromstring(
            result["best_state"], dtype=np.int8, sep=" ")
        family_id = result["family_id"]
        family = families[family_id]
        chrom = result["chrom"]
        pos = result["pos"]
        pos_end = result["pos_end"]

        expected_ploidy = np.asarray([
            get_interval_locus_ploidy(
                chrom, pos, pos_end, p.sex, genome
            ) for p in family.members_in_order
        ])
        if variant_type == Allele.Type.large_duplication:
            alt_row = actual_ploidy - expected_ploidy
        elif variant_type == Allele.Type.large_deletion:
            alt_row = expected_ploidy - actual_ploidy
        else:
            raise ValueError(
                f"unexpected variant type: {variant_type}")
        ref_row = expected_ploidy - alt_row
        best_state = np.stack((ref_row, alt_row)).astype(np.int8)
        result["best_state"] = best_state

        return result

    return transformer

def _cnv_person_id_to_best_state(
        families: FamiliesData, genome: ReferenceGenome) \
        -> Callable[[dict[str, Any]], dict[str, Any]]:
    """Transform variant into canonical family id/best state form.

    In case the genotype is specified by person id having the variant
    this transformer will transform it into canonical family id/best state
    form
    """
    def transformer(result: dict[str, Any]) -> dict[str, Any]:
        person_id = result["person_id"]
        if "family_id" in result:
            family_id = result["family_id"]
            family = families[family_id]
            person = family.persons[person_id]
        else:
            assert len(families.persons_by_person_id[person_id]) == 1
            person = families.persons_by_person_id[person_id][0]
            family = families[person.family_id]

        chrom = result["chrom"]
        pos = result["pos"]
        pos_end = result["pos_end"]

        expected_ploidy = np.asarray([
            get_interval_locus_ploidy(
                chrom, pos, pos_end, p.sex, genome
            ) for p in family.members_in_order
        ])
        alt_row = np.zeros(len(family.members_in_order), dtype=np.int8)
        alt_row[person.member_index] = 1

        ref_row = expected_ploidy - alt_row
        best_state = np.stack((ref_row, alt_row)).astype(np.int8)
        result["best_state"] = best_state
        result["family_id"] = family.family_id

        return result

    return transformer

def _configure_cnv_best_state(
        header: list[str],
        transformers: list[Callable[[dict[str, Any]], dict[str, Any]]],
        families: FamiliesData,
        genome: ReferenceGenome,
        cnv_person_id: Optional[str] = None,
        cnv_family_id: Optional[str] = None,
        cnv_best_state: Optional[str] = None) -> None:
    """Configure header and transformers that handle CNV family genotypes."""
    if cnv_person_id is not None:
        # if cnv_family_id is not None and cnv_best_state is not None:
        #     raise ValueError(
        #         f"mixed configuration of cnv best state: "
        #         f"person_id({cnv_person_id}) <-> "
        #         f"family_id({cnv_family_id}) and "
        #         f"best_state({cnv_best_state})"
        #     )
        person_index = header.index(cnv_person_id)
        header[person_index] = "person_id"

        transformers.append(
            _cnv_person_id_to_best_state(families, genome)
        )
    else:
        if cnv_family_id is None:
            cnv_family_id = "family_id"
        if cnv_best_state is None:
            cnv_best_state = "best_state"

        family_index = header.index(cnv_family_id)
        best_state_index = header.index(cnv_best_state)

        header[family_index] = "family_id"
        header[best_state_index] = "best_state"

        transformers.append(
            _cnv_dae_best_state_to_best_state(families, genome))

def _cnv_variant_to_variant_type(
    cnv_plus_values: Optional[list[str]] = None,
    cnv_minus_values: Optional[list[str]] = None
) -> Callable[[dict[str, Any]], dict[str, Any]]:
    """Transform variant type to canonical internal representation.

    This transformer is used to transform variant type to canonical
    inernal representation using :class:`Allele.Type.large_duplication` and
    :class:`Allele.Type.large_deletion`.
    """
    if cnv_minus_values is None:
        cnv_minus_values = ["CNV-"]
    if cnv_plus_values is None:
        cnv_plus_values = ["CNV+"]

    def transformer(result: dict[str, Any]) -> dict[str, Any]:
        variant = result["variant"]
        if variant in cnv_plus_values:
            variant_type = Allele.Type.large_duplication
        elif variant in cnv_minus_values:
            variant_type = Allele.Type.large_deletion
        else:
            raise ValueError(f"unexpected CNV variant type: {variant}")

        result["variant_type"] = variant_type
        return result

    return transformer

def _configure_cnv_variant_type(
    header: list[str],
    transformers: list[Callable[[dict[str, Any]], dict[str, Any]]],
    cnv_variant_type: Optional[str] = None,
    cnv_plus_values: Optional[Union[str, list[str]]] = None,
    cnv_minus_values: Optional[Union[str, list[str]]] = None
) -> None:
    """Configure header and transformer needed to handle CNV variant type."""
    if cnv_plus_values is None:
        cnv_plus_values = ["CNV+"]
    elif isinstance(cnv_plus_values, str):
        cnv_plus_values = [cnv_plus_values]

    if cnv_minus_values is None:
        cnv_minus_values = ["CNV-"]
    if isinstance(cnv_minus_values, str):
        cnv_minus_values = [cnv_minus_values]

    if cnv_variant_type is None:
        cnv_variant_type = "variant"

    variant_type_index = header.index(cnv_variant_type)
    header[variant_type_index] = "variant"

    transformers.append(
        _cnv_variant_to_variant_type(cnv_plus_values, cnv_minus_values)
    )

def _configure_loader(
        header: list[str],
        families: FamiliesData,
        genome: ReferenceGenome,
        cnv_chrom: Optional[str] = None,
        cnv_start: Optional[str] = None,
        cnv_end: Optional[str] = None,
        cnv_location: Optional[str] = None,
        cnv_person_id: Optional[str] = None,
        cnv_family_id: Optional[str] = None,
        cnv_best_state: Optional[str] = None,
        cnv_variant_type: Optional[str] = None,
        cnv_plus_values: Optional[list[str]] = None,
        cnv_minus_values: Optional[list[str]] = None) \
        -> tuple[
            list[str],
            list[Callable[[dict[str, Any]], dict[str, Any]]]]:
    """Configure all headers and transformers needed to handle CNVs input."""
    # pylint: disable=too-many-arguments
    transformers: list[
        Callable[[dict[str, Any]], dict[str, Any]]] = []
    header = deepcopy(header)

    _configure_cnv_location(
        header, transformers,
        cnv_chrom, cnv_start, cnv_end,
        cnv_location
    )

    _configure_cnv_variant_type(
        header, transformers,
        cnv_variant_type, cnv_plus_values, cnv_minus_values)

    _configure_cnv_best_state(
        header, transformers,
        families, genome,
        cnv_person_id,
        cnv_family_id, cnv_best_state)

    return header, transformers

def flexible_cnv_loader(
    filepath_or_buffer: Union[str, Path, TextIO],
    families: FamiliesData,
    genome: ReferenceGenome,
    cnv_chrom: Optional[str] = None,
    cnv_start: Optional[str] = None,
    cnv_end: Optional[str] = None,
    cnv_location: Optional[str] = None,
    cnv_person_id: Optional[str] = None,
    cnv_family_id: Optional[str] = None,
    cnv_best_state: Optional[str] = None,
    cnv_variant_type: Optional[str] = None,
    cnv_plus_values: Optional[list[str]] = None,
    cnv_minus_values: Optional[list[str]] = None,
    cnv_sep: str = "\t",
    **_kwargs: Any
) -> Generator[dict[str, Any], None, None]:
    """Load variants from CNVs input and transform them into DataFrames.

    This function uses flexible variant loader infrastructure to
    load variants from a CNVs data input and transform them into a pandas
    `DataFrame`.
    """
    # pylint: disable=too-many-locals,too-many-arguments
    def line_splitter(line: str) -> list[str]:
        return line.strip("\n\r").split(cnv_sep)

    if isinstance(filepath_or_buffer, (str, Path)):
        infile = open(filepath_or_buffer, "rt")
    else:
        infile = filepath_or_buffer  # type: ignore

    with infile as infile:
        # FIXME don't throw StopIteration and fix the next line
        line = next(infile)  # pylint: disable=stop-iteration-return
        header = line_splitter(line)

        header, transformers = _configure_loader(
            header,
            families,
            genome,
            cnv_chrom,
            cnv_start,
            cnv_end,
            cnv_location,
            cnv_person_id,
            cnv_family_id,
            cnv_best_state,
            cnv_variant_type,
            cnv_plus_values,
            cnv_minus_values)

        variant_generator = flexible_variant_loader(
            infile, header, line_splitter, transformers,
            filters=[]
        )

        for record in variant_generator:
            yield record


from lingpy import *
from collections import defaultdict
from sinopy import sinopy
import re

csv1 = csv2list('2017-02-18-Behr-1-197-draft-2-western.csv', strip_lines=False)
csv2 = csv2list('2017-02-18-Behr-1-197-draft-2-eastern.csv', strip_lines=False)

chars = defaultdict(list)

for i, line in enumerate(csv2[1:]+csv1[1:]):
    if len(line) > 7:
        if '+' in line[5] or '>' in line[5] or '{' in line[5] or len(line[5].strip()) > 1:
            chars[line[5]] += [(i+1, line[2], line[3])]

guessed = 0
with open('characters.md', 'w') as f:
    f.write('## Problematic Chars in Behr 2008\n\n')
    f.write('number | first page | character | conversion | occurrences | pages \n')
    f.write('--- | --- | --- | --- | --- | --- | --- \n')
    for i, (char, vals) in enumerate(sorted(chars.items(), key=lambda x:
        x[1][0][0])):
        guess = '?'
        if len(char) == 3:
            guess = sinopy.character_from_structure(char[1]+char[0]+char[2])
            if guess != '?':
                guessed += 1
        else:
            _guess = char
            composed = re.findall('{(.*?)}', char)
            for c in composed:
                if len(c) == 3:
                    r = sinopy.character_from_structure(c[1]+c[0]+c[2])
                    if r != '?':
                        _guess = _guess.replace('{'+c+'}', r)
            if _guess != char:
                guessed += 1
                if len(_guess) == 3:
                    guess2 = sinopy.character_from_structure(_guess[1]+_guess[0]+_guess[2])
                    if guess2 != '?':
                        guess = guess2
                    else:
                        guess = _guess
        f.write('{0} | {1} | {2} | {3} | {4} | {5} \n'.format(
            i+1,
            vals[0][1],
            char,
            guess,
            len(vals),
            ', '.join(['{0}: {1}'.format(y, z) for x, y, z in vals])
            ))


#import libraries

import os
import string
import glob
import subprocess
from subprocess import Popen
import shutil
import time
import datetime
from datetime import datetime
import pandas as pd
import requests
import tkinter as tk
from tkinter import *
from tkinter import ttk
from tkinter import messagebox
from PIL import ImageTk, Image
import plotly.graph_objects as go
from plotly.subplots import make_subplots

#import packages

from packages import tool_path, action

def get_taxonomy_file():
    df_taxonomy = pd.read_csv(tool_path.get_current_path()[0] +'/packages/ressources/taxonomy_by_DB.tsv', sep = '\t') # Read specific taxonomy found in LOTUS_DB
    df_taxonomy_all = pd.read_csv(tool_path.get_current_path()[0] + '/packages/ressources/all_taxonomy_DB.tsv', sep = '\t') # Read general taxonomy found in LOTUS_DB
    return df_taxonomy, df_taxonomy_all

def get_chemontology_file():
    df_chemontology = pd.read_csv(tool_path.get_current_path()[0] + '/packages/ressources/NPClassifier_taxonomy.tsv', sep = '\t')
    return df_chemontology

# Main and most important function to scrap LOTUS
def get_lotus(IDD):

    ############################ Get the code for IDD
    TorC = IDD.split(' : ')[0]
    if TorC == 'T':
        TaxoDB = IDD.split(' : ')[1]
        NPClassifier = ''
        level = IDD.split(' : ')[2]
        name = IDD.split(' : ')[3]
    elif TorC == 'C' :
        TaxoDB = IDD.split(' : ')[1]
        NPClassifier = IDD.split(' : ')[2]
        level = IDD.split(' : ')[3]
        name = IDD.split(' : ')[4]

    elif TorC == 'LTS':
        TaxoDB = IDD.split(' : ')[1]
        NPClassifier = ''
        level = ''
        name = IDD.split(' : ')[2]

    elif TorC == 'F':
        TaxoDB = IDD.split(' : ')[1]
        NPClassifier = ''
        level = ''
        name = IDD.split(' : ')[2]

    print(TorC, TaxoDB, NPClassifier, level, name)

    url = str('https://lotus.naturalproducts.net/api/search/simple?query=' + str(name))

    response = requests.get(url)
    storage = [response.json()]  #get the seaerch result in a json format

    nb = len(str(storage)[1:-1].split("{'id'")) - 1   #get the number of molecule for the search
    print(str(nb) + ' results' )

    ###################################Get all the keys for the research

    list_key_value=[]
    for mol in range(nb):
        for key in storage[0]['naturalProducts'][mol]['taxonomyReferenceObjects'] :

            if TaxoDB != 'All_Taxonomy_DB' and TorC == 'T':
                if TaxoDB in storage[0]['naturalProducts'][mol]['taxonomyReferenceObjects'][key]:
                    for go in range(len(storage[0]['naturalProducts'][mol]['taxonomyReferenceObjects'][key][TaxoDB])):
                        if storage[0]['naturalProducts'][mol]['taxonomyReferenceObjects'][key][TaxoDB][go][level] != None and name in str(storage[0]['naturalProducts'][mol]['taxonomyReferenceObjects'][key][TaxoDB][go][level]):
                            key_value = [mol,key, TaxoDB, go]
    #                     print(key_value)
                            list_key_value.append(key_value)
    #                     print(storage[0]['naturalProducts'][key_value[0]]['taxonomyReferenceObjects'][key_value[1]][TaxoDB][0])

            elif TaxoDB == 'All_Taxonomy_DB' and TorC == 'T':
                for taxo in storage[0]['naturalProducts'][mol]['taxonomyReferenceObjects'][key]:
                    for go in range(len(storage[0]['naturalProducts'][mol]['taxonomyReferenceObjects'][key][taxo])):
                        if storage[0]['naturalProducts'][mol]['taxonomyReferenceObjects'][key][taxo][go][level] != None and name in str(storage[0]['naturalProducts'][mol]['taxonomyReferenceObjects'][key][taxo][go][level]):
                            key_value = [mol,key, taxo, go]
    #                     print(key_value)
                            list_key_value.append(key_value)
    #                     print(storage[0]['naturalProducts'][key_value[0]]['taxonomyReferenceObjects'][key_value[1]][key_value[2]][0])

            elif TaxoDB != 'All_Taxonomy_DB' and TorC == 'C':
                if TaxoDB in storage[0]['naturalProducts'][mol]['taxonomyReferenceObjects'][key]:
                    for go in range(len(storage[0]['naturalProducts'][mol]['taxonomyReferenceObjects'][key][TaxoDB])):
                        if storage[0]['naturalProducts'][mol][level] != None and name in str(storage[0]['naturalProducts'][mol][level]):
                            key_value = [mol,key,TaxoDB, go]
    #                     print(key_value)
                            list_key_value.append(key_value)

            elif TaxoDB == 'All_Taxonomy_DB' and TorC == 'C':
                for taxo in storage[0]['naturalProducts'][mol]['taxonomyReferenceObjects'][key]:
                    for go in range(len(storage[0]['naturalProducts'][mol]['taxonomyReferenceObjects'][key][taxo])):
                        if storage[0]['naturalProducts'][mol][level] != None and name in str(storage[0]['naturalProducts'][mol][level]):
                            key_value = [mol,key, taxo, go]
    #                     print(key_value)
                            list_key_value.append(key_value)

            elif TaxoDB != 'All_Taxonomy_DB' and TorC == 'LTS':
                if TaxoDB in storage[0]['naturalProducts'][mol]['taxonomyReferenceObjects'][key]:
                    for go in range(len(storage[0]['naturalProducts'][mol]['taxonomyReferenceObjects'][key][TaxoDB])):
                        key_value = [mol,key,TaxoDB, go]
                        list_key_value.append(key_value)

            elif TaxoDB == 'All_Taxonomy_DB' and TorC == 'LTS':
                for taxo in storage[0]['naturalProducts'][mol]['taxonomyReferenceObjects'][key]:
                    for go in range(len(storage[0]['naturalProducts'][mol]['taxonomyReferenceObjects'][key][taxo])):
                        key_value = [mol,key, taxo, go]
                        list_key_value.append(key_value)

            elif TaxoDB != 'All_Taxonomy_DB' and TorC == 'F':
                if TaxoDB in storage[0]['naturalProducts'][mol]['taxonomyReferenceObjects'][key]:
                    for go in range(len(storage[0]['naturalProducts'][mol]['taxonomyReferenceObjects'][key][TaxoDB])):
                        key_value = [mol,key, TaxoDB, go]
                        list_key_value.append(key_value)

            elif TaxoDB == 'All_Taxonomy_DB' and TorC == 'F':
                for taxo in storage[0]['naturalProducts'][mol]['taxonomyReferenceObjects'][key]:
                    for go in range(len(storage[0]['naturalProducts'][mol]['taxonomyReferenceObjects'][key][taxo])):
                        key_value = [mol,key, taxo, go]
                        list_key_value.append(key_value)

    ######################## transform the keys to search
    df = pd.DataFrame(list_key_value, columns =['mol', 'key', 'taxo', 'go'])
    list_new_key_value = []
    new_list = list(set(df.mol))
    for mol in new_list:
        AA = {}
        list_new_key = set(df.loc[df.mol == mol].key.to_list())

        df2 = df.loc[df.mol == mol]
        aa = {}
        for x in list_new_key:

            list_taxo = list(set(df2.loc[df2.key == x].taxo.to_list()))

            df3 = df2.loc[df2.key == x]

            rst={}
            for y in list_taxo:

                list_go = df3.loc[df3.taxo == y].go.to_list()

                rst[y] = list_go
            aa[x] = rst
        AA = [mol, aa]

        list_new_key_value.append(AA)

    ########################### get Lotus
    global Dict
    Dict = {
            'Lotus_ID' :{} , 'smiles' : {}, 'inchi' : {}, 'inchikey' : {},  'cas' : {}, 'iupac_name' : {}, 'molecular_formula' : {}, 'molecular_weight' : {}, 'xlogp' : {} ,
            'superkingdom':{}, 'kingdom':{}, 'phylum':{}, 'classx':{}, 'order':{}, 'family': {}, 'genus' : {}, 'species' :{} ,
            'chemicalTaxonomyClassyfireKingdom' : {} , 'chemicalTaxonomyClassyfireSuperclass' :{}, 'chemicalTaxonomyClassyfireClass' : {} , 'chemicalTaxonomyClassyfireDirectParent' : {} ,
            'chemicalTaxonomyNPclassifierPathway' : {} ,'chemicalTaxonomyNPclassifierSuperclass' : {} , 'chemicalTaxonomyNPclassifierClass' : {},
            'traditional_name' : {}
           } #Create a global dict to store metadata from LOTUS

    xx = 1  #start dict at 1

    # if TaxoDB != "All_Taxonomy_DB" and TorC == 'T':
    zz = 0
    for molindex in list_new_key_value:

        Dict['Lotus_ID'][xx] = storage[0]['naturalProducts'][molindex[0]]['lotus_id']
        Dict['smiles'][xx] = storage[0]['naturalProducts'][molindex[0]]['smiles']
        Dict['inchi'][xx] = storage[0]['naturalProducts'][molindex[0]]['inchi']
        Dict['inchikey'][xx] = storage[0]['naturalProducts'][molindex[0]]['inchikey']
        Dict['cas'][xx] = storage[0]['naturalProducts'][molindex[0]]['cas']
        Dict['traditional_name'][xx] = storage[0]['naturalProducts'][molindex[0]]['traditional_name'].encode(encoding = 'UTF-8', errors = 'replace')
        Dict['iupac_name'][xx] = (storage[0]['naturalProducts'][molindex[0]]['iupac_name']).encode(encoding = 'UTF-8', errors = 'replace')
        Dict['molecular_formula'][xx] = storage[0]['naturalProducts'][molindex[0]]['molecular_formula']
        Dict['molecular_weight'][xx] = storage[0]['naturalProducts'][molindex[0]]['molecular_weight']
        Dict['xlogp'][xx] = storage[0]['naturalProducts'][molindex[0]]['xlogp']

        Dict['chemicalTaxonomyClassyfireKingdom'][xx] = storage[0]['naturalProducts'][molindex[0]]['chemicalTaxonomyClassyfireKingdom']
        Dict['chemicalTaxonomyClassyfireSuperclass'][xx] = storage[0]['naturalProducts'][molindex[0]]['chemicalTaxonomyClassyfireSuperclass']
        Dict['chemicalTaxonomyClassyfireClass'][xx] = storage[0]['naturalProducts'][molindex[0]]['chemicalTaxonomyClassyfireClass']
        Dict['chemicalTaxonomyClassyfireDirectParent'][xx] = storage[0]['naturalProducts'][molindex[0]]['chemicalTaxonomyClassyfireDirectParent']
        Dict['chemicalTaxonomyNPclassifierPathway'][xx] = storage[0]['naturalProducts'][molindex[0]]['chemicalTaxonomyNPclassifierPathway']
        Dict['chemicalTaxonomyNPclassifierSuperclass'][xx] = storage[0]['naturalProducts'][molindex[0]]['chemicalTaxonomyNPclassifierSuperclass']
        Dict['chemicalTaxonomyNPclassifierClass'][xx] = storage[0]['naturalProducts'][molindex[0]]['chemicalTaxonomyNPclassifierClass']

        spk1={}
        ki1={}
        ph1={}
        cl1={}
        or11={}
        fm1={}
        ge1={}
        spe1={}

        print(molindex[0])
        for key in list_new_key_value[zz][1]: ####################decalage de 1 !!!!!!!! a reparer
            print(key)
            spk={}
            ki={}
            ph={}
            cl={}
            or1={}
            fm={}
            ge={}
            spe={}

            for taxo in list_new_key_value[zz][1][key]:
                print(taxo)
                superkingdom=[]
                kingdom=[]
                phylum=[]
                classx=[]
                order=[]
                family=[]
                genus=[]
                species=[]
                for go in range(len(list_new_key_value[zz][1][key][taxo])):

                    if storage[0]['naturalProducts'][molindex[0]]['taxonomyReferenceObjects'][key][taxo][list_new_key_value[zz][1][key][taxo][go]]['superkingdom'] not in superkingdom:
                        superkingdom.append(storage[0]['naturalProducts'][molindex[0]]['taxonomyReferenceObjects'][key][taxo][list_new_key_value[zz][1][key][taxo][go]]['superkingdom'])

                    if storage[0]['naturalProducts'][molindex[0]]['taxonomyReferenceObjects'][key][taxo][list_new_key_value[zz][1][key][taxo][go]]['kingdom'] not in kingdom:
                        kingdom.append(storage[0]['naturalProducts'][molindex[0]]['taxonomyReferenceObjects'][key][taxo][list_new_key_value[zz][1][key][taxo][go]]['kingdom'])

                    if storage[0]['naturalProducts'][molindex[0]]['taxonomyReferenceObjects'][key][taxo][list_new_key_value[zz][1][key][taxo][go]]['phylum'] not in phylum:
                        phylum.append(storage[0]['naturalProducts'][molindex[0]]['taxonomyReferenceObjects'][key][taxo][list_new_key_value[zz][1][key][taxo][go]]['phylum'])

                    if storage[0]['naturalProducts'][molindex[0]]['taxonomyReferenceObjects'][key][taxo][list_new_key_value[zz][1][key][taxo][go]]['classx'] not in classx:
                        classx.append(storage[0]['naturalProducts'][molindex[0]]['taxonomyReferenceObjects'][key][taxo][list_new_key_value[zz][1][key][taxo][go]]['classx'])

                    if storage[0]['naturalProducts'][molindex[0]]['taxonomyReferenceObjects'][key][taxo][list_new_key_value[zz][1][key][taxo][go]]['order'] not in order:
                        order.append(storage[0]['naturalProducts'][molindex[0]]['taxonomyReferenceObjects'][key][taxo][list_new_key_value[zz][1][key][taxo][go]]['order'])

                    if storage[0]['naturalProducts'][molindex[0]]['taxonomyReferenceObjects'][key][taxo][list_new_key_value[zz][1][key][taxo][go]]['family'] not in family:
                        family.append(storage[0]['naturalProducts'][molindex[0]]['taxonomyReferenceObjects'][key][taxo][list_new_key_value[zz][1][key][taxo][go]]['family'])

                    if storage[0]['naturalProducts'][molindex[0]]['taxonomyReferenceObjects'][key][taxo][list_new_key_value[zz][1][key][taxo][go]]['genus'] not in genus:
                        genus.append(storage[0]['naturalProducts'][molindex[0]]['taxonomyReferenceObjects'][key][taxo][list_new_key_value[zz][1][key][taxo][go]]['genus'])

                    if storage[0]['naturalProducts'][molindex[0]]['taxonomyReferenceObjects'][key][taxo][list_new_key_value[zz][1][key][taxo][go]]['species'] not in species:
                        species.append(storage[0]['naturalProducts'][molindex[0]]['taxonomyReferenceObjects'][key][taxo][list_new_key_value[zz][1][key][taxo][go]]['species'])
    #                 print(species)

                spk[taxo] = ' - '.join([str(e) for e in superkingdom])
                ki[taxo] =' - '.join([str(e) for e in kingdom])
                ph[taxo] =' - '.join([str(e) for e in phylum])
                cl[taxo]=' - '.join([str(e) for e in classx])
                or1[taxo]=' - '.join([str(e) for e in order])
                fm[taxo]=' - '.join([str(e) for e in family])
                ge[taxo]=' - '.join([str(e) for e in genus])
                spe[taxo]=' - '.join([str(e) for e in species])

                if taxo in spk1:
                    spk1[taxo]=spk1[taxo] + ' - ' + spk[taxo]
                    ki1[taxo]=ki1[taxo] +' - ' + ki[taxo]
                    ph1[taxo]=ph1[taxo] +' - ' + ph[taxo]
                    cl1[taxo]=cl1[taxo] +' - ' + cl[taxo]
                    or11[taxo]=or11[taxo] +' - ' + or1[taxo]
                    fm1[taxo]= fm1[taxo] +' - ' + fm[taxo]
                    ge1[taxo]=ge1[taxo] +' - ' + ge[taxo]
                    spe1[taxo]=spe1[taxo] +' - ' + spe[taxo]
                else:
                    spk1[taxo]= spk[taxo]
                    ki1[taxo]=ki[taxo]
                    ph1[taxo]=ph[taxo]
                    cl1[taxo]=cl[taxo]
                    or11[taxo]=or1[taxo]
                    fm1[taxo]=fm[taxo]
                    ge1[taxo]=ge[taxo]
                    spe1[taxo]=spe[taxo]

        for taxo in spk1:
            spk1[taxo] = set(spk1[taxo].split(' - '))
            ki1[taxo] = set(ki1[taxo].split(' - '))
    #         print(ki[taxo])
            ph1[taxo] = set(ph1[taxo].split(' - '))
            cl1[taxo] = set(cl1[taxo].split(' - '))
            or11[taxo] = set(or11[taxo].split(' - '))
            fm1[taxo] = set(fm1[taxo].split(' - '))
            ge1[taxo] = set(ge1[taxo].split(' - '))
            spe1[taxo] = set(spe1[taxo].split(' - '))

        print(spe1)

        Dict['superkingdom'][xx] = spk1
        Dict['kingdom'][xx] =ki1
        Dict['phylum'][xx] =ph1
        Dict['classx'][xx] = cl1
        Dict['order'][xx]  = or11
        Dict['family'][xx] = fm1
        Dict['genus'][xx] = ge1
        Dict['species'][xx] = spe1
            #################################################################################
        zz=zz+1
    #         Dict['reference_wikidata_id'][xx] = storage[0]['naturalProducts'][molindex[0]]['taxonomyReferenceObjects'][molindex[1]][molindex[2]][0]['reference_wikidata_id']

        xx = xx + 1 # Next molecule

    global df_lotus
    df_lotus = pd.DataFrame.from_dict(Dict)
    #df_lotus.to_csv('cfmid_input.tsv', sep = '\t', index = False)

    return df_lotus

# Function to search all the selected listed criteria in LOTUS_DB
def get_lotus_add():

    list_IDD = action.get_selected_search_criteria_list()

    if list_IDD[0] == '':

        print(str(datetime.now()) + " Come on, make a choice ! ")
        messagebox.showinfo("Info", "Come on, make a chocie !")
    else :

        Dict_neutral = {
                'Lotus_ID' :{} , 'smiles' : {}, 'inchi' : {}, 'inchikey' : {},  'cas' : {}, 'iupac_name' : {}, 'molecular_formula' : {}, 'molecular_weight' : {}, 'xlogp' : {} ,
                'superkingdom':{}, 'kingdom':{}, 'phylum':{}, 'classx':{}, 'order':{}, 'family': {}, 'genus' : {}, 'species' :{} ,
                'chemicalTaxonomyClassyfireKingdom' : {} , 'chemicalTaxonomyClassyfireSuperclass' :{}, 'chemicalTaxonomyClassyfireClass' : {} , 'chemicalTaxonomyClassyfireDirectParent' : {} ,
                'chemicalTaxonomyNPclassifierPathway' : {} ,'chemicalTaxonomyNPclassifierSuperclass' : {} , 'chemicalTaxonomyNPclassifierClass' : {}
                ,'traditional_name' : {}
        }
        df_general = pd.DataFrame.from_dict(Dict_neutral)

        for IDD in list_IDD:
            #get_lotus(IDD.split(' : ')[1])
            print(IDD)
            get_lotus(IDD)

            df_general = pd.concat([df_general, df_lotus], ignore_index = True)
            df_general= df_general.drop_duplicates(subset=['Lotus_ID'])
        df_general.to_csv(tool_path.get_current_path()[0]+ '/LOTUS_DB_input/cfmid_input.tsv', sep = '\t', index = False)
        fileout = open(tool_path.get_current_path()[0] + '/LOTUS_DB_input/cfmid_input.txt', "w")
        fileout_sirius_db = open(tool_path.get_current_path()[0] + '/LOTUS_DB_input/structural_db.txt', "w") #####230228
        print(str(datetime.now()) + " There are " + str(len(df_general['Lotus_ID'])) +" molecules for " + str(list_IDD))

        messagebox.showinfo("Info", "There are " + str(len(df_general['Lotus_ID'])) +" molecules for " + str(list_IDD))
        for i in df_general['Lotus_ID']:
             fileout.write(i + ' ' + df_general['smiles'][int(str(df_general.Lotus_ID[df_general.Lotus_ID == i ].index.to_list()[0]))] + '\n' )
             fileout_sirius_db.write(df_general['smiles'][int(str(df_general.Lotus_ID[df_general.Lotus_ID == i ].index.to_list()[0]))] + ' ' + i + '\n' ) ######230228
        fileout.close()
        fileout_sirius_db.close() #####230228
    errorlog = open(tool_path.get_current_path()[0] + '/log/' + str(datetime.now()).replace(' ', '_').replace(':', '_').replace('.','_') + '_input_criteria.txt', 'w')
    errorlog.write("This log contains criteria for research in the online LOTUS DataBase on the "+ str(datetime.now()) + " ." + "\n")
    errorlog.write('The research has been made with the "for all selected categories" method.' + "\n")
    errorlog.write("There are " + str(len(df_general['Lotus_ID'])) +" molecules corresponding to your criteria." + "\n")
    for IDD in list_IDD:
        errorlog.write(str(IDD) + "\n")
    errorlog.close()

# Function to search combined selected listed criteria in LOTUS_DB
def get_lotus_or():

    list_IDD = action.get_selected_search_criteria_list()
    if list_IDD[0] == '':
        print(str(datetime.now()) + " Come on, make a choice ! ")
        messagebox.showinfo("Info", "Come on, make a chocie !")
    else :

        Dict_neutral = {
                'Lotus_ID' :{} , 'smiles' : {}, 'inchi' : {}, 'inchikey' : {},  'cas' : {}, 'iupac_name' : {}, 'molecular_formula' : {}, 'molecular_weight' : {}, 'xlogp' : {} ,
                'superkingdom':{}, 'kingdom':{}, 'phylum':{}, 'classx':{}, 'order':{}, 'family': {}, 'genus' : {}, 'species' :{} ,
                'chemicalTaxonomyClassyfireKingdom' : {} , 'chemicalTaxonomyClassyfireSuperclass' :{}, 'chemicalTaxonomyClassyfireClass' : {} , 'chemicalTaxonomyClassyfireDirectParent' : {} ,
                'chemicalTaxonomyNPclassifierPathway' : {} ,'chemicalTaxonomyNPclassifierSuperclass' : {} , 'chemicalTaxonomyNPclassifierClass' : {}
                ,'traditional_name' : {}
        }

        df_general = pd.DataFrame.from_dict(Dict_neutral)
        df_general2 = pd.DataFrame.from_dict(Dict_neutral)

        selected_tax = []
        selected_chemical = []
        selected_formula = []
        for IDD in list_IDD:
            if 'T : ' in IDD:
                selected_tax.append(IDD) # Is the criteria a Taxonomic criteria ?
            elif 'C : ' in IDD:
                #selected_chemical.append(IDD.split(' : ')[4]) # Is the criteria a Chemontologic criteria
                selected_chemical.append(IDD)
            elif 'F : ' in IDD:
                #selected_formula.append(IDD.split(' : ')[2]) # Is the criteria a Formula criteria ?
                selected_formula.append(IDD)
        if selected_tax != [] :         # if taxonomy criteria exist
            for sel_tax in selected_tax:

                get_lotus(sel_tax)

                if selected_chemical != [] : #if taxonomy and chemontoly criteria exist
                    for sel_chemical in selected_chemical:
                        if sel_chemical.split(' : ')[4] in df_lotus['chemicalTaxonomyNPclassifierPathway'].to_list():
                            new_df = df_lotus.loc[(df_lotus['chemicalTaxonomyNPclassifierPathway'] == str(sel_chemical.split(' : ')[4])) | (df_lotus['chemicalTaxonomyNPclassifierPathway'] == str(str(sel_chemical.split(' : ')[4]) + '|' + str(sel_chemical.split(' : ')[4])))] ####230228
                            df_general = pd.concat([df_general, new_df], ignore_index = True)

                        elif sel_chemical.split(' : ')[4] in df_lotus['chemicalTaxonomyNPclassifierSuperclass'].to_list():
                            new_df = df_lotus.loc[(df_lotus['chemicalTaxonomyNPclassifierSuperclass'] == str(sel_chemical.split(' : ')[4]))]
                            df_general = pd.concat([df_general, new_df], ignore_index = True)

                        elif sel_chemical.split(' : ')[4] in df_lotus['chemicalTaxonomyNPclassifierClass'].to_list():
                            new_df = df_lotus.loc[(df_lotus['chemicalTaxonomyNPclassifierClass'] == str(sel_chemical.split(' : ')[4]))]
                            df_general = pd.concat([df_general, new_df], ignore_index = True)

                        if selected_formula != [] :       #if taxonomy and chemontoly and formula criteria exist
                            for sel_formula in selected_formula:
                                if sel_formula.split(' : ')[2] in df_general['molecular_formula'].to_list():
                                    new_df = df_general.loc[(df_general['molecular_formula'] == str(sel_formula.split(' : ')[2]))]
                                    df_general2 = pd.concat([df_general2, new_df], ignore_index = True)
                        else :
                            df_general2 = pd.concat([df_general2, df_general], ignore_index = True) #if taxonomy and chemontoly exist but not formula

                else : #if taxonomy exist but not chemontology
                    if selected_formula != [] : #if taxonomy and formula criteria exist but not chemontology
                        for sel_formula in selected_formula:
                            if sel_formula.split(' : ')[2] in df_lotus['molecular_formula'].to_list():
                                new_df = df_lotus.loc[(df_lotus['molecular_formula'] == str(sel_formula.split(' : ')[2]))]
                                df_general2 = pd.concat([df_general2, new_df], ignore_index = True)
                    else :   # if only taxonomy criteria exist
                        df_general2 = pd.concat([df_general2, df_lotus], ignore_index = True)

        else :    #if taxonomy criteria does not exist
            if selected_chemical != [] :  #if taxonomy criteria does not exist but chemontology exist
                for sel_chemical in selected_chemical:
                    get_lotus(sel_chemical)
                    if selected_formula != [] :    #if taxonomy criteria does not exist but chemontology and formula exist
                        for sel_formula in selected_formula:

                            if sel_formula.split(' : ')[2] in df_lotus['molecular_formula'].to_list():
                                new_df = df_lotus.loc[(df_lotus['molecular_formula'] == str(sel_formula.split(' : ')[2]))]
                                df_general2 = pd.concat([df_general2, new_df], ignore_index = True)
                    else :   #if taxonomy criteria and formula do not exist but chemontology exist
                        df_general2 = pd.concat([df_general2, df_lotus], ignore_index = True)
            else :  #if taxonomy criteria and chemontoly do not exist
                if selected_formula != [] :   #if taxonomy criteria and chemontoly do not exist but formula exist
                    for sel_formula in selected_formula:
                        get_lotus(sel_formula)
                        df_general2 = pd.concat([df_general2, df_lotus], ignore_index = True)

        #messagebox.showinfo("Info", "There are " + str(len(df_general2['Lotus_ID'])) +" molecules for this research")
        df_general2= df_general2.drop_duplicates(subset=['Lotus_ID']) #delete duplicates in the results of the search
        if df_general2['Lotus_ID'].to_list():
            df_general2.to_csv(tool_path.get_current_path()[0]+ '/LOTUS_DB_input/cfmid_input.tsv', sep = '\t', index = False)  #export metadata in .TSV file

            fileout = open(tool_path.get_current_path()[0]+ '/LOTUS_DB_input/cfmid_input.txt', "w") #export LOTUS_ID and SMILES in text file
            fileout_sirius_db = open(tool_path.get_current_path()[0] + '/LOTUS_DB_input/structural_db.txt', "w") #####230228
            for i in df_general2['Lotus_ID']:
                 fileout.write(i + ' ' + df_general2['smiles'][int(str(df_general2.Lotus_ID[df_general2.Lotus_ID == i ].index.to_list()[0]))] + '\n' )
                 fileout_sirius_db.write(df_general2['smiles'][int(str(df_general2.Lotus_ID[df_general2.Lotus_ID == i ].index.to_list()[0]))] + ' ' + i + '\n' ) ######230228
            fileout.close()
            fileout_sirius_db.close()
            print(str(datetime.now()) + " There is a total of " + str(len(df_general2['Lotus_ID'].to_list()))  + " molecules for requested categories.")
            messagebox.showinfo("Info", "There is a total of " + str(len(df_general2['Lotus_ID'].to_list()))  + " molecules for requested categories.")
        else:
            messagebox.showinfo("Info", "There is no molecules for requested categories.")
            print(str(datetime.now()) + " Info", "There is no molecules for requested categories.")

    errorlog = open(tool_path.get_current_path()[0] + '/log/' + str(datetime.now()).replace(' ', '_').replace(':', '_').replace('.','_') + '_input_criteria.txt', 'w')
    errorlog.write("This log contains criteria for research in the online LOTUS DataBase on the "+ str(datetime.now()) + " ." + "\n")
    errorlog.write('The research has been made with the "for selected chemical class.es in selected family.ies" method.' + "\n")
    errorlog.write("There are " + str(len(df_general2['Lotus_ID'])) +" molecules corresponding to your criteria." + "\n")
    for IDD in list_IDD:
        errorlog.write(str(IDD) + "\n")
    errorlog.close()

#get the cfmid 4.0 docker image identifier to log
def get_cfmid_docker_image():
    ID_container_id = open(tool_path.get_current_path()[0] + '/CFM_ID_4/ID_container_cfmid.txt', 'r').read().split('\n')[0]
    return ID_container_id


#!/usr/bin/env python3
"""
Tests for the `pre_commit.git` submodule.
"""
from os import path
from tempfile import TemporaryDirectory
from unittest import (
    main,
    TestCase
)

from pre_commit.git import (
    ForbiddenCharacterError,
    GitHandle,
    RepositoryError
)

from tests.util import BasicRepo

class TestGitHandle(TestCase):

    def test_get_git_root(self):

        try:
            # initialize repo
            repo = BasicRepo()

            # initialize git handle
            git_handle = GitHandle(repo.repo_path)

            # test
            self.assertEqual(
                path.abspath(git_handle.root),
                path.abspath(repo.repo_path)
            )

        except Exception:
            raise

        finally:
            repo.delete()

    def test_check_path_is_allowed(self):

        try:
            # initialize repo
            repo = BasicRepo()

            # initialize git handle
            git_handle = GitHandle(repo.repo_path)

            # test some bad paths and file names...
            with self.assertRaises(ForbiddenCharacterError):
                git_handle._check_path_is_allowed("a bad/path")

            with self.assertRaises(ForbiddenCharacterError):
                git_handle._check_path_is_allowed("/a\nice/path/")

            with self.assertRaises(ForbiddenCharacterError):
                git_handle._check_path_is_allowed("this_is_/'the'/_path")

            with self.assertRaises(ForbiddenCharacterError):
                git_handle._check_path_is_allowed(
                    path.join("\\", "a_path")
                )

            with self.assertRaises(ForbiddenCharacterError):
                git_handle._check_path_is_allowed(
                    path.join("/some/path", "this is_bad.ext")
                )

        except Exception:
            raise

        finally:
            repo.delete()

    def test_get_head_hash_bare(self):

        try:
            # initialize repo
            repo = BasicRepo()

            # initialize git handle
            git_handle = GitHandle(repo.repo_path)

            # test
            self.assertEqual(
                git_handle.get_head_hash(),
                "4b825dc642cb6eb9a060e54bf8d69288fbee4904"
            )

        except Exception:
            raise

        finally:
            repo.delete()

    def test_get_head_hash_not_bare(self):

        try:
            # initialize repo
            repo = BasicRepo(bare=False)

            # initialize git handle
            git_handle = GitHandle(repo.repo_path)

            # test commit
            test_commit_file = path.join(repo.repo_path, "test_commit")
            with open(test_commit_file, "w") as foo:
                foo.write("test commit")
            repo.repo.git.add(test_commit_file)
            repo.repo.git.commit("-m", "test commit")

            self.assertEqual(
                git_handle.get_head_hash(),
                repo.repo.head.commit.hexsha
            )

        except Exception:
            raise

        finally:
            repo.delete()

    def test_get_staged_file_paths(self):

        try:
            # initialize repo
            repo = BasicRepo(bare=False)

            # initialize git handle
            git_handle = GitHandle(repo.repo_path)

            # test staging
            test_staged_file = path.join(repo.repo_path, "test_staged_file")
            with open(test_staged_file, "w") as foo:
                foo.write("test stage")

            test_staged_file1 = path.join(repo.repo_path, "test_staged_file1")
            with open(test_staged_file1, "w") as foo:
                foo.write("test stage1")

            # stage
            repo.repo.git.add([test_staged_file, test_staged_file1])

            self.assertEqual(
                set(git_handle.get_staged_files_paths()),
                set([
                    path.relpath(test_staged_file, repo.repo_path),
                    path.relpath(test_staged_file1, repo.repo_path)
                ])
            )

        except Exception:
            raise

        finally:
            repo.delete()

    def test_staged_file_content(self):

        try:
            # initialize repo
            repo = BasicRepo(bare=False)

            # initialize git handle
            git_handle = GitHandle(repo.repo_path)

            # write file
            test_staged_file = path.join(repo.repo_path, "test_staged_file")
            with open(test_staged_file, "w") as foo:
                foo.write("test stage")

            # stage
            repo.repo.git.add([test_staged_file])

            # edit the file again
            with open(test_staged_file, "a") as foo:
                foo.write("\nmore")

            # get staged file path
            pth = git_handle.get_staged_files_paths()[0]

            self.assertEqual(
                git_handle.get_staged_file_content(pth),
                b"test stage"
            )

        except Exception:
            raise

        finally:
            repo.delete()

class TestGitHandleErrors(TestCase):

    def test_repository_error(self):
        try:
            tmp = TemporaryDirectory()
            with self.assertRaises(RepositoryError):
                GitHandle(tmp.name)

        except Exception:
            raise

        finally:
            tmp.cleanup()

if __name__ == "__main__":
    main()


from flask import Flask, jsonify, request

app = Flask(__name__)

accounts = [
    {"name":"Billy", 'balance':457.74},
    {"name":"Renesmee", 'balance':-150.0},
    {"name":"Edward", 'balance':4156.9},
    {"name":"Marla", 'balance':321.31},
    {"name":"Andrew", 'balance':-120.1},
    {"name":"Roxane", 'balance':-10.2},
    {'name':'Kelly','balance':250.0}
]

groups =[
    {"name":"Positive", "members":[]},
    {"name":"Negative","members":[]}
]

def groupsFormat():
    i = 0
    for gr in groups:
        gr['members']=[]

    for person in accounts:
        if person['balance']>=0:
            groups[0]['members'].append(i)
        else:
            groups[1]['members'].append(i)
        i += 1

@app.route("/",methods=["GET"])
def wellcome():
    return "Wellcome to banking system"

@app.route("/accounts",methods=["GET"])
def getAccounts():
    return jsonify(accounts)

@app.route("/accounts/<id>",methods=["GET"])
def getAccount(id):
    id = int(id)-1
    return jsonify(accounts[id])

@app.route("/accounts/<id>",methods=["DELETE"])
def deleteAccount(id):
    id = int(id)-1
    ac = accounts[id]
    accounts.remove(ac)
    groupsFormat()
    return jsonify(ac)

@app.route("/accounts",methods=["POST"])
def addAccounts():
    name = request.json['name']
    balance = float(request.json['balance'])
    data = {'name':name,'balance':balance}
    accounts.append(data)

    groupsFormat()

    return jsonify(data)

@app.route("/accounts",methods=["PUT"])
def changeAccount():
    name = request.json['name']
    balance = float(request.json['balance'])
    data = {'name':name,'balance':balance}
    for acc in accounts:
        if acc['name'] == name:
            acc['balance'] = balance
            return jsonify(data)

    groupsFormat()

    return "No such name"

@app.route("/groups",methods=["GET"])
def getGroups():
    change_groups = []
    for g in groups:
        change_groups.append({"name":g["name"]})
        for acc in g["members"]:
            #str = "member"+str(acc)
            change_groups[-1].update({"member"+str(acc):accounts[acc]})
    return jsonify(change_groups)

if __name__ == "__main__":
    groupsFormat()
    app.run(port = 8080)


import fire,os,sys
import numpy as np
from tqdm import tqdm
from skimage import io,morphology
from keras.utils import to_categorical
import cv2
from ulitities.base_functions import get_file,send_message_callback,load_label
def post_process_segment(inf,outf,Flag_cv=True, minsize=10, area_threshold=1000):
    # pass
    # import cv2
    img = load_label(inf)
    print(np.unique(img))
    NB = len(np.unique(img))-1
    if Flag_cv:
        kernel = np.ones((minsize,minsize),np.uint8)
        opening = cv2.morphologyEx(img, cv2.MORPH_OPEN, kernel)
        closing = cv2.morphologyEx(opening, cv2.MORPH_CLOSE, kernel)
        cv2.imwrite(outf,closing)
    else:
        # NB=1
        if NB >1:
            NB+=1
            try:
                # label= to_categorical(img,num_classes=NB,dtype='uint8')
                label = to_categorical(img)
            except:
                print("Failed when to transform one hot")
                return -1
            result_list = []
            for i in range(NB):
                t = remove_small_objects_and_holes(label[:, :, i], minsize*10, area_threshold, True)
                result_list.append(t[:,:,None])

            label = np.concatenate(result_list, axis=2)
            label = np.argmax(label, axis=2).astype(np.uint8)
        else:
            label =img
            label= remove_small_objects_and_holes(label, minsize*10, area_threshold, True)
            label = np.asarray(label,np.uint8)
        cv2.imwrite(outf, label)

def remove_small_objects_and_holes(label,min_size, area_threshold, in_place=True):
    label=morphology.remove_small_objects(label==1,min_size=min_size, connectivity=1,in_place=in_place)
    label=morphology.remove_small_holes(label==1,area_threshold=area_threshold,connectivity=1,in_place=in_place)
    return label

def batch_rmovesmallobj(send_message_callback,inputdir,outputdir,flag_cv=True,msize=5, thd=1000):
    if not os.path.isdir(inputdir):
        send_message_callback("Please check input directory:{}".format(inputdir))
        sys.exit(-1)
    if not os.path.isdir(outputdir):
        send_message_callback('Warning: output directory is not existed')
        os.mkdir(outputdir)
    files,_=get_file(inputdir)
    for file in files:#tqdm(files):
        send_message_callback("Dealing with : "+file)
        absname = os.path.split(file)[1]
        outputfile = os.path.join(outputdir, absname)
        post_process_segment(file, outputfile,Flag_cv=flag_cv, minsize=msize, area_threshold=thd)
if __name__ == "__main__":
    # batch_rmovesmallobj(r"C:\Users\SCRS\Pictures\111",r"C:\Users\SCRS\Pictures\222")
    # inputfile='/home/omnisky/PycharmProjects/data/samples/isprs/train_ori/label_all/top_potsdam_7_13.tif'
    # outputfile='/media/omnisky/e0331d4a-a3ea-4c31-90ab-41f5b0ee2663/traindata/scrs_building/test/crfss/whole48-tt.tif'
    # post_process_segment(inputfile,outputfile, Flag_cv=False, minsize=10)

    fire.Fire()


import numpy as np
from matplotlib import pyplot as plt
import matplotlib.colors
import random as rand

class Board:
    def __init__(self,dim,targetTerrain): #set default values for a board
        self.board = np.zeros((dim,dim), dtype= float)
        self.target = (rand.randint(0,dim-1),rand.randint(0,dim-1))
        self.dim = dim
        self.populateTerrain()
        self.newTarget(targetTerrain)

    def populateTerrain(self):
        terrain = [.1, .3, .7, .9]
        for i in range(self.dim):
            for j in range(self.dim):
                self.board[i,j] = rand.choices(terrain,weights = (.2, .3, .3, .2))[0]

    def newTarget(self,terrain):
        while True:
            x,y = (rand.randint(0,self.dim-1),rand.randint(0,self.dim-1))
            if self.board[x,y] == terrain:
                self.target = (x,y)
                break

    def printBoard(self):
        #print(self.board)
        plt.figure(figsize = (10,10))
        plt.pcolor(self.board,edgecolors = "black", cmap = 'Set3', linewidths = 1)
        for (j,i),label in np.ndenumerate(self.board): #consider using a mapping for label to mine/flag
            plt.text(i,j,label,ha='left',va='bottom')
        plt.tight_layout()
        plt.show() #block=False
        #plt.pause(.2)
        #plt.close()


"""
Aprenda a manipular datas
Realizar conversao de texto para data e vice-versa
realizar soma e subtracao em datas
- Como recuperar a data atual(DATE)
- Como trabalhar com a data, alterando sua formatação
- Como gerar um horário(TIME)
- Retornar data e hora atual(DATETIME)
- Alterar formação do DATETIME
- Realizar soma e subtracao de datas com TIMEDELTA
"""

from datetime import date, time, datetime, timedelta

def trabalhando_date():
    data_atual = date.today()
    print(data_atual.strftime('%d/%m/%Y')) # Dia - número, m - Mês, - Y - Ano com 4 digitos # strftime - vira string
    print(data_atual.strftime('%A %B %Y')) # Dia - nome, m - Mês, Y - Ano com 4 digitos # strftime - vira string

def trabalhando_time():
    time_atual = time(hour=15, minute=10, second=30)
    print(time_atual)
    print(time_atual.strftime('%H:%M:%S')) # H - Hora, M - Minuto, S - segundo # strftime - vira string

def trabalhando_datetime():
    data_atual = datetime.now()
    print(data_atual)
    print(data_atual.strftime('%d/%m/%Y - %H:%M:%S')) # d - Dia número, m - Mês numero, Y - Ano numero # strftime - vira string
    print(data_atual.strftime('%c')) # c - Dia da semana nome, Mês nome, Dia numero, Hora:minuto:segundos, ano 4 digitos  # strftime - vira string
    tupla = ('Segunda-feira', 'Terça-feira', 'Quarta-feira', 'Quinta-feira', 'Sexta-feira', 'Sabado', 'Domingo')
    print(tupla[data_atual.weekday()])
    data_string = '01/01/2019 12:20:25'
    data_convertida = datetime.strptime(data_string, '%d/%m/%Y %H:%M:%S')
    print(data_convertida)
    nova_data = data_convertida - timedelta(days=365)
    print(nova_data)

if __name__ == '__main__':
    trabalhando_date()
    trabalhando_time()
    trabalhando_datetime()


from nltk.corpus import stopwords

from settings.common import word_tf_df
from preprocessing_pipeline.NextGen import NextGen
from preprocessing_pipeline import (Preprocess, RemovePunctuation, Capitalization, RemoveStopWords,
                                    RemoveShortWords, TwitterCleaner, RemoveUrls)

def load_flat_dataset(path):
    dataset = []
    with open(path, 'r') as f:
        for line in f:
            dataset.append(line.strip().split(' '))
    return dataset

def load_dataset_with_dates(path):
    dataset = []
    with open(path, 'r') as f:
        for line in f:
            dataset.append(line.strip().split('\t')[1].split(' '))
    return dataset

if __name__ == '__main__':
    dataset_names = ['sample_tweets']
    forbidden_words = []  # list of words to be blacklisted [amp, rt, ...]
    syn_file = None  # synonym file containing line for each set of synonyms: [word], [synonym1], [synonym2], ...
    extra_ngrams = []  # list of $-separated ngrams: [new$york$city, joe$biden, donald$trump, no$new$taxes]

    for j in range(0, len(dataset_names)):
        ds = dataset_names[j]

        stopwords_list = stopwords.words('english')
        stopwords_list.append(['rt', 'amp'])

        pipeline = Preprocess()
        rp = RemovePunctuation(keep_hashtags=False)
        ru = RemoveUrls()
        cap = Capitalization()
        short_words = RemoveShortWords()
        tc = TwitterCleaner()
        rsw = RemoveStopWords(extra_sw=stopwords_list)

        pipeline.document_methods = [(tc.remove_deleted_tweets, str(tc),),
                                     (tc.remove_users, str(tc),),
                                     (ru.remove_urls, str(ru),),
                                     (rp.remove_punctuation, str(rp),),
                                     (cap.lowercase, str(cap),),
                                     (tc.remove_rt, str(tc),),
                                     (rsw.remove_stopwords, str(rsw),),
                                     (short_words.remove_short_words, str(short_words),)
                                     ]

        ng = NextGen()
        path = 'data/{}.csv'.format(ds)
        dataset = load_dataset_with_dates(path)
        processed_dataset = ng.full_preprocess(dataset, pipeline, ngram_min_freq=10, extra_bigrams=None, extra_ngrams=extra_ngrams)

        with open('data/{}_lightweight.csv'.format(ds), 'w') as f:
            for i in range(0, len(processed_dataset)):
                doc = processed_dataset[i]
                f.write('{}\n'.format(' '.join(doc)))

        freq = {}
        freq = word_tf_df(freq, processed_dataset)
        processed_dataset = ng.filter_by_tfidf(dataset=processed_dataset, freq=freq, threshold=0.25)

        with open('data/{}_lightweight_tdidf.csv'.format(ds), 'w') as f:
            for i in range(0, len(processed_dataset)):
                doc = processed_dataset[i]
                f.write('{}\n'.format(' '.join(doc)))


import os
from music21 import humdrum
from music21 import converter
import chant21

class MultipleSpinesException(Exception):
    """An exception raised when encountering multiple spines
    while expecting only 1"""
    pass

def extract_phrases_from_spine(spine):
    """Enxtract the phrases as music21 streams from kern
    Phrases in a humdrum file are delimited by curly brackets.
    All events that do not occur between an opening and closing bracket
    (i.e., are not in a phrase) are ignored.

    Parameters
    ----------
    spine : music21.humdrum.spineParser.KernSpine
        A humdrum spine with the melody

    Returns
    -------
    list
        A list of phrases, each represented as a music21.stream.Stream
    """
    phrases = []
    in_phrase = False
    cur_phrase = humdrum.spineParser.KernSpine()
    for event in spine:
        is_phrase_start = event.contents.startswith('{')
        is_phrase_end = event.contents.endswith('}')

        if is_phrase_start:
            in_phrase = True

        if in_phrase:
            cur_phrase.append(event)

        if is_phrase_end and in_phrase:
            phrases.append(cur_phrase)
            cur_phrase = humdrum.spineParser.KernSpine()
            in_phrase = False

    # Parse spines and return streams
    for phrase in phrases: phrase.parse()
    streams = [phrase.stream.flat for phrase in phrases]
    return streams

def extract_phrases_from_kern_file(filename: str) -> list:
    """Extract all phrases from a kern file.
    Phrases are returned as flattened music21 streams

    Parameters
    ----------
    filename : string
        filename of the file

    Returns
    -------
    list
        List of phrases as music21.stream.Stream objects

    Raises
    ------
    MultipleSpinesException
        Whenever the file contains multiple spines
    """
    song = humdrum.parseFile(filename)
    if len(song.spineCollection.spines) > 1:
        raise MultipleSpinesException
    spine = song.spineCollection.spines[0]
    return extract_phrases_from_spine(spine)

def extract_phrases_from_gabc_file(filename: str) -> list:
    chant = converter.parse(filename)
    return chant.phrases

def extract_phrases_from_file(filepath: str) -> list:
    """Extract phrases from a file to a list of music21 streams.
    Delegates the extration based on the file extension.

    Parameters
    ----------
    filename : string
        filename of the file

    Returns
    -------
    list
        List of phrases as music21.stream.Stream objects
    """
    if not os.path.exists(filepath):
        raise FileNotFoundError
    extension = os.path.splitext(filepath)[1]
    if extension == '.krn':
        return extract_phrases_from_kern_file(filepath)
    elif extension == '.gabc':
        return extract_phrases_from_gabc_file(filepath)
    else:
        raise Exception('Unknown file format')


import os
import time
import threading
from dotenv import load_dotenv
import pyttsx3
import data_handler
from datetime import datetime
from datetime import timedelta
from task import Task

class AssistantApp:
    def __init__(self):
        # Initialize text to speech
        self.engine = pyttsx3.init()

        # Initialize tasks
        self.tasks = []
        self.task_id = 1
        self.handler = data_handler.DataHandler()

    def get_response(self, message):
        response = self.handler.ChatGPT(message)

        # Say the response with TTS
        print(response)
        self.engine.say(response)
        self.engine.runAndWait()

    def add_task(self):
        text = input("Enter task: ")
        repeats = input("Generate repeating task? yes/no: ")

        repeat_interval = None

        if repeats == 'yes':
            repeat_interval = int(input("Enter repeat interval (days): "))
            repeats = True
        else:
            repeats = False

        for attempt in range(3):
            try:
                date = input("Enter date dd:mm:yy: ")
                time = input("Enter time hh:mm: ")

                date_list = date.split(":")
                time_list = time.split(":")
                task_datetime = datetime(int(date_list[2]) + 2000, int(date_list[1]), int(date_list[0]), int(time_list[0]), int(time_list[1]))
                break
            except Exception as e:
                print("Incorrect time format, try again.")
                continue

        if repeat_interval == None:
            self.tasks.append(Task(self.task_id, text, task_datetime, repeats))
        else:
            self.tasks.append(Task(self.task_id, text, task_datetime, repeats, repeat_interval))

        self.task_id += 1

        # Store task to tasks.csv
        self.handler.update_tasks(self.tasks)
        return self.task_id - 1

    def delete_task(self, delete_id):

        for task in self.tasks:
            if task.task_id == delete_id:
                self.tasks.remove(task)

        self.handler.update_tasks(self.tasks)

    def print_tasks(self):
        if self.tasks == None:
            print("No tasks assigned")
            return

        print('*******************************************')

        for task in self.tasks:
            if task.task_reminder_added == True:
                print(f'{task.task_id}: REMINDER: {task.task_text} {task.task_datetime.strftime("%Y-%m-%d %H:%M")}')
            else:
                print(f'{task.task_id}: {task.task_text} {task.task_datetime.strftime("%Y-%m-%d %H:%M")}')

        print('*******************************************')

    def load_tasks(self):
        tasks = self.handler.load_tasks()

        if tasks == None or len(tasks) == 0:
            return

        self.tasks = tasks

        last_id = tasks[len(tasks) - 1].task_id
        self.task_id = last_id + 1

    def check_tasks(self):
        # format: (1, 'test', datetime.datetime(2023, 5, 15, 15, 0))

        while True:
            current_time = datetime.now()

            if self.tasks == None:
                return

            for task in self.tasks:
                if current_time.year == task.task_datetime.year and current_time.month == task.task_datetime.month and current_time.day == task.task_datetime.day and current_time.hour == task.task_datetime.hour and current_time.minute == task.task_datetime.minute:
                    print(task.task_text)
                    self.engine.say(f"You have a task that requires your attention. The task is {task.task_text}")
                    self.engine.runAndWait()

                    if task.task_repeats == True and task.task_repeat_added == False:
                        self.add_repeat_task(task)
                        task.task_repeat_added = True

                    if task.task_reminder_added == False:
                        task.task_datetime = task.task_datetime + timedelta(hours=1)
                        task.task_reminder_added = True

            # Possibly reduntant update tasks
            self.handler.update_tasks(self.tasks)
            time.sleep(45)

    def add_repeat_task(self, task):
        task_datetime = task.task_datetime + timedelta(days=task.task_repeat_interval)

        self.tasks.append(Task(self.task_id, task.task_text, task_datetime, task.task_repeats, task.task_repeat_interval))
        self.task_id += 1

        self.handler.update_tasks(self.tasks)

    def add_reminder(self, task):
        #self.handler.add_reminder()
        pass

    def clear_reminders(self):
        #self.handler.clear_reminders()
        pass

    def main(self):
        # Load tasks from tasks.csv
        self.load_tasks()

        # Start a thread to continuously check tasks
        check_tasks_thread = threading.Thread(target=self.check_tasks)
        check_tasks_thread.daemon = True  # Allow the thread to exit when the main program exits
        check_tasks_thread.start()

        while True:

            self.print_tasks()

            print("Choose an option or use text to prompt ChatGPT")
            print("0: exit")
            print("1: add task")
            print("2: delete task")
            print("3: list tasks")
            print("-------------------------------------------")
            option = input("prompt: ")

            if option == "0":
                self.handler.update_tasks(self.tasks)
                break
            elif option == "1":
                self.add_task()
            elif option == "2":
                delete_id = int(input("Enter task id: "))
                self.delete_task(delete_id)
            elif option == "3":
                continue
            else:
                self.get_response(option)

if __name__ == "__main__":
    assistant = AssistantApp()
    assistant.main()


from datetime import datetime

import requests
from flask import Flask, render_template

app = Flask(__name__)

year = datetime.now().year
# print(year)
response = requests.get('https://api.npoint.io/362a61befce3d173e925')
blog_posts = response.json()

# print(blog_posts)

@app.route('/')
def home():
    return render_template("index.html", post_data=blog_posts, date=year)

@app.route('/post/<int:index>')
def post_selected(index):
    post_current = None
    for post in blog_posts:
        if post['id'] == index:
            post_current = post

    return render_template("post.html", post_data=post_current, date=year)

if __name__ == "__main__":
    app.run(debug=True)


# Importing Required Libraries
import cv2
# define a class called ImageReader
class ImageReader:
    def __init__(self, filename):
        self.filename = filename

    def read_image(self):
        try:
            img = cv2.imread(self.filename)
            if img is None:
                raise Exception("Error: Unable to read image file")
        except Exception as e:
            print(str(e))
        else:
            return img

# create an instance of the ImageReader
reader = ImageReader(r"C:\PYTHON\AI_ML\OpenCV\Computer_vision.jpeg")
# call the read_image() method
img = reader.read_image()
if img is not None:
    # to show an image
    cv2.imshow("Image", img)
    # to dispaly upto defined time
    cv2.waitKey(0)
    # to close the open window
    cv2.destroyAllWindows()
# After executing this code seperate window will be open with an image which has given in the path.

# ### Reading an Image by allow the user to enter a link

# Importing Required Libraries
import cv2
import urllib.request
import numpy as np

# define a class called ImageReader
class ImageReader:
    def __init__(self, url):
        self.url = url

    def read_image(self):
        try:
            resp = urllib.request.urlopen(self.url)
            img = np.asarray(bytearray(resp.read()), dtype="uint8")
            img = cv2.imdecode(img, cv2.IMREAD_COLOR)
            if img is None:
                raise Exception("Error: Unable to read image file")
        except Exception as e:
            print(str(e))
        else:
            return img

# create an instance of the ImageReader
url = input("Enter the URL of the image file: ")
reader = ImageReader(url)
img = reader.read_image()
if img is not None:
    cv2.imshow("Image", img)
    cv2.waitKey(0)
    cv2.destroyAllWindows()
# After Executing above code It will ask you to enter an url link to diaplay an image.


from collections import deque
import sys
input = lambda: sys.stdin.readline().rstrip()

def bfs(x, y):
    q = deque()
    q.append((x, y))
    field[x][y] = 0
    while (q):
        a, b = q.popleft()
        for i in range(8):
            nx = a + dx[i]
            ny = b + dy[i]
            if 0 <= nx < m and 0 <= ny < n and field[nx][ny] == 1:
                field[nx][ny] = 0
                q.append((nx, ny))

if __name__ == '__main__':
    dx = [-1, -1, -1, 0, 0, 1, 1, 1]
    dy = [-1, 0, 1, -1, 1, -1, 0, 1]
    while True:
        n, m = map(int, input().split())
        cnt = 0
        if n == 0 and m == 0:
            break
        field = [list(map(int, input().split())) for _ in range(m)]
        for i in range(m):
            for j in range(n):
                if field[i][j] == 1:
                    bfs(i, j)
                    cnt += 1
        print(cnt)


import tweepy
import json
import time
import datetime
import ConfigParser
import tweepy
from tweepy.streaming import StreamListener

'''
Get API keys from Configurations document
'''
config = ConfigParser.ConfigParser()
config.readfp(open(r'./configurations.txt'))

consumerKey=config.get('API Keys', 'consumerKey')
consumerSecret=config.get('API Keys', 'consumerSecret')
accessToken=config.get('API Keys', 'accessToken')
accessSecret=config.get('API Keys', 'accessSecret')

'''
Returns given number of tweets for given Twitter handle
'''
def get_tweets(twitter_handle, num_tweets):

    auth = tweepy.AppAuthHandler(consumerKey, consumerSecret)
    api = tweepy.API(auth, wait_on_rate_limit=True, wait_on_rate_limit_notify=True)

    timeline = api.user_timeline(screen_name=twitter_handle, count=num_tweets)

    user_tweets = []

    for current_tweet in timeline:
        if not current_tweet.retweeted:
            tweet = {}
            tweet['tweetId'] = current_tweet.id
            tweet['message'] = current_tweet.text
            tweet['author'] = current_tweet.user.name
            s = str(current_tweet.created_at)
            new_s = time.mktime(datetime.datetime.strptime(s, "%Y-%m-%d %H:%M:%S").timetuple()) * 1000
            tweet['timestamp'] = new_s
            user_tweets.append(tweet)

    return user_tweets


import cv2
import numpy as np
import os
os.environ["OPENCV_IO_ENABLE_OPENEXR"]="1"
from PIL import Image
from utils.metrics_nocs import align, prepare_data_posefitting, draw_detections
from datasets.datasets import exr_loader,load_meta
syn_depth_path = '/data/sensor/data/real_data/test_0/0000_gt_depth.exr'
nocs_path = '/data/sensor/data/real_data/test_0/0000_coord.png'
mask_path = '/data/sensor/data/real_data/test_0/0000_mask.png'
meta_path = '/data/sensor/data/real_data/test_0/0000_meta.txt'
obj_dir = '/data/sensor/data/cad_model/real_cat_known'
synset_names = ['other',  # 0
                    'bottle',  # 1
                    'bowl',  # 2
                    'camera',  # 3
                    'can',  # 4
                    'car',  # 5
                    'mug',  # 6
                    'aeroplane',  # 7
                    'BG',  # 8
                    ]
intrinsics = np.zeros((3,3))

img_h = 720
img_w = 1280
fx = 918.295227050781
fy = 917.5439453125

cx = img_w * 0.5 - 0.5
cy = img_h * 0.5 - 0.5
camera_params = {
    'fx': fx,
    'fy': fy,
    'cx': cx,
    'cy': cy,
    'yres': img_h,
    'xres': img_w,
}
hw = (640 / img_w, 360 / img_h)
camera_params['fx'] *= hw[0]
camera_params['fy'] *= hw[1]
camera_params['cx'] *= hw[0]
camera_params['cy'] *= hw[1]
camera_params['xres'] *= hw[0]
camera_params['yres'] *= hw[1]
intrinsics[0,0] = camera_params['fx']
intrinsics[0,2] = camera_params['cx']
intrinsics[1,1] = camera_params['fy']
intrinsics[1,2] = camera_params['cy']
intrinsics[2,2] = 1.0

_syn_depth =  cv2.imread(syn_depth_path, cv2.IMREAD_ANYCOLOR | cv2.IMREAD_ANYDEPTH)
if len(_syn_depth.shape) == 3:
    _syn_depth = _syn_depth[:, :, 0]
coords = Image.open(nocs_path).convert('RGB')
coords = np.array(coords) / 255.

if mask_path.split('.')[-1] == 'exr':
    _mask = exr_loader(mask_path, ndim=1)
else:
    _mask = Image.open(mask_path)
    _mask = np.array(_mask)
if mask_path.split('.')[-1] == 'exr':
    _mask = np.array(_mask * 255, dtype=np.int32)
_meta = load_meta(meta_path)

_mask_sem = np.full(_mask.shape, 8) #, 0）
_scale = np.ones((10,3)) #(class_num+1,3)
for i in range(len(_meta)):
    _mask_sem[_mask == _meta[i]["index"]] = _meta[i]["label"] #1
    if _meta[i]["instance_folder"] !=" " :

        bbox_file = os.path.join(obj_dir,_meta[i]["instance_folder"] ,_meta[i]["name"],'bbox.txt')
        bbox = np.loadtxt(bbox_file)
        _scale[_meta[i]["label"]] = bbox[0, :] - bbox[1, :]
    else :
        _scale[_meta[i]["label"]] = np.ones((3))
    _scale[_meta[i]["label"]] /= np.linalg.norm(_scale[_meta[i]["label"]])

gt_class_ids , gt_scores , gt_masks ,gt_coords ,\
                    gt_boxes = prepare_data_posefitting(_mask_sem,coords)

result = {}
result['gt_RTs'], scales, error_message, _ = align(gt_class_ids,
                                                        gt_masks,
                                                        gt_coords,
                                                        _syn_depth,
                                                        intrinsics,
                                                        synset_names)

if 1:
    output_path = 'tmp'
    draw_rgb = False
    save_dir =os.path.join(output_path ,'save_{}'.format(i))
    if not os.path.exists(save_dir) :
        os.mkdir(save_dir)
    data = 'camera'
    result['gt_handle_visibility'] = np.ones_like(gt_class_ids)
    rgb_path = '/data/sensor/data/real_data/test_0/0000_color.png'
    _rgb = Image.open(rgb_path).convert('RGB')
    _rgb = _rgb.resize((640,360))
    _rgb = np.array(_rgb)

    draw_detections(_rgb, save_dir, data, 1, intrinsics, synset_names, draw_rgb,
                            gt_boxes, gt_class_ids, gt_masks, gt_coords, result['gt_RTs'], scales, np.ones(gt_boxes.shape[0]),
                            gt_boxes, gt_class_ids, gt_masks, gt_coords, result['gt_RTs'], np.ones(gt_boxes.shape[0]), scales)


import json
import time
import datetime
import os.path

import GlobalConstants
import GlobalEnums
import Tools

from os import path

class ChatManager:
	"""
	Chat Manager
	"""
	def __init__(self):
		#self.chatChannelGlobal = {}
		self.createChannel(GlobalEnums.ChatChannel.CHAT_CHANNEL_GLOBAL)

	def createChannel(self, channelType):
		if channelType == GlobalEnums.ChatChannel.CHAT_CHANNEL_GLOBAL:
			self.chatChannelGlobal = {}

	def appendMessage(self, channelType, sender, message):
		createdMessageObj = {
			'sender': sender,
			'message': message,
			'date': Tools.getDate(),
			'time': Tools.getTime()
		}
		if channelType == GlobalEnums.ChatChannel.CHAT_CHANNEL_GLOBAL:
			index = len(self.chatChannelGlobal) + 1 # so it is not a zwro index
			self.chatChannelGlobal[index] = createdMessageObj
		# Each message added, make sure we aren't over our chat limit.
		self.channelGarbageCollector()

	def updateMessage(self, channelType, sender, message):
		createdMessageObj = {
			'sender': sender,
			'message': message,
			'date': Tools.getDate(),
			'time': Tools.getTime()
		}
		if channelType == GlobalEnums.ChatChannel.CHAT_CHANNEL_GLOBAL:
			self.chatChannelGlobal.update(createdMessageObj)  # use this only for existing values

	def channelGarbageCollector(self):
		if len(self.chatChannelGlobal) >= GlobalConstants.CHAT_CHANNEL_GLOBAL_CAPACITY:
			self.writeAndClear(GlobalEnums.ChatChannel.CHAT_CHANNEL_GLOBAL)

	def writeAndClear(self, channelType):
		self.writeChannelTypeToFile(channelType)
		if channelType == GlobalEnums.ChatChannel.CHAT_CHANNEL_GLOBAL:
			self.chatChannelGlobal.clear()

	def writeChannelTypeToFile(self, channelType):
		dictToWrite = None
		if channelType == GlobalEnums.ChatChannel.CHAT_CHANNEL_GLOBAL:
			dictToWrite = self.chatChannelGlobal
		self.writeChannelToFile(dictToWrite, channelType)

	def writeChannelToFile(self, dict, channelType):
		print(	os.path.abspath(os.getcwd()))
		fileName = "%s.%s.%s" % (str(channelType.name), Tools.getDateDots(), Tools.getTimeDots())
		# This is a testing check and should not happen in the real world, basically this will
		# track many messages coming in at once and overwriting the last wrote file
		dupeIndex = 0
		while path.exists('ChatLogs/%s.txt' % fileName):
			dupeIndex += 1
			fileName = "%s.%s.%s.%s" % (str(channelType.name), Tools.getDateDots(), Tools.getTimeDots(), dupeIndex)
		with open('ChatLogs/%s.txt' % fileName, 'w') as file:
			file.write(json.dumps(dict, indent=2))  # use `json.loads` to do the reverse, indent adds to new line


import torch
import pandas as pd
import numpy as np
import torch.nn as nn
import statistics
from DLDUlib import device, train, optimize_ols, center, normalize, r_squared, cross_validate_train
import copy

names = ['SalePrice','1st_Flr_SF','2nd_Flr_SF','Lot_Area','Overall_Qual',
    'Overall_Cond','Year_Built','Year_Remod/Add','BsmtFin_SF_1','Total_Bsmt_SF',
    'Gr_Liv_Area','TotRms_AbvGrd','Bsmt_Unf_SF','Full_Bath']
df = pd.read_csv('AmesHousing.csv', names = names)
data = df.values # read data into a numpy array (as a list of lists)
data = data[1:] # remove the first list which consists of the labels
data = data.astype(np.float) # coerce the entries in the numpy array to floats
data = torch.FloatTensor(data) # convert data to a Torch tensor

data_train, means = center(data)
data_train, stdevs = normalize(data)

class NonLinearModel(nn.Module):

  def __init__(self):
    super(NonLinearModel, self).__init__()
    self.layer1 = nn.Linear(13, 10)
    self.layer2 = nn.Linear(10, 1)

  def forward(self, xss):
    xss = self.layer1(xss)
    xss = torch.relu(xss)
    return self.layer2(xss)

model = NonLinearModel()
criterion = nn.MSELoss()

epochs = 60
learning_rate = 0.005
momentum = 0.9
batchsize = 20

folds = 10
bail_after = 10
no_improvement = 0
best_valids = 1e15*torch.ones(folds)

while no_improvement < bail_after:

  model, valids = cross_validate_train(
      k = folds,
      model = model,
      criterion = criterion,
      features = data_train[:,1:],
      targets = data_train[:,:1],
      epochs = epochs,
      learning_rate = learning_rate,
      momentum = momentum,
      batchsize = batchsize,
      verbosity = 1
  )

  print(best_valids.mean().item())
  if valids.mean().item() < best_valids.mean().item():
    best_model = copy.deepcopy(model)
    best_valids = valids
    no_improvement = 0
  else:
    no_improvement += 1
    print("no improvement", no_improvement)

test_input = torch.FloatTensor([2855, 0, 26690, 8, 7, 1952, 1972, 1040, 2080, 1756, 8, 841, 2])

test_input = (test_input - means[1:]) / stdevs[1:]

test_output = best_model.forward(test_input).data[0]*stdevs[0]+means[0]

print("test_output:", test_output)


## Code to make a PCR reaction using a pre-mixed master-mix, distribute it between wells
## of a 96-well plate, add primers and add a given number of template samples.

###INPUT### PCR variables
num_replicates = 8
num_templates = 4

total_PCR_volume = 20
master_mix_volume = 8
template_volume = 1
primer_volume = 2.5

#Define primer pairs

from opentrons import robot, containers, instruments

#Define containers - source_tube rack = cold block
pcr_plate = containers.load('96-PCR-flat', 'B1', 'pcr-plate')
trash = containers.load('trash-box', 'A3')
p200rack = containers.load('tiprack-200ul', 'A1', 'p200-rack')

#Create 96-well trough
containers.create(
	'96-well-square',
	grid=(8,12),
	spacing=(9,9),
	diameter=7,
	depth=40
)

mix_trough = containers.load('96-well-square', 'B2', 'mix-trough')

#Create 6x12 p20 tip rack container
containers.create(
	'tiprack-200ul-6x12',
	grid=(6,12),
	spacing=(9, 9),
	diameter=5,
	depth=60
)

p20rack = containers.load('tiprack-200ul-6x12', 'E1', 'p20-rack')

#Create 3x6 2ml tube rack for DNA samples
containers.create(
	'3x6-tube-rack-2ml',
	grid=(3,6),
	spacing=(19.5,19.5),
	diameter=9.5,
	depth=40
)

template_primer_rack = containers.load('3x6-tube-rack-2ml', 'B3', 'template-primer-rack')
cold_rack = containers.load('3x6-tube-rack-2ml', 'D3', 'cold-rack')

#Define pipettes
p20 = instruments.Pipette(
    trash_container=trash,
    tip_racks=[p20rack],
    min_volume=2,
    max_volume=20,
    axis="a"
)

p200 = instruments.Pipette(
	trash_container=trash,
	tip_racks=[p200rack],
	min_volume=20,
	max_volume=200,
	axis="b"
)

#Define locations of PCR components
water_source = cold_rack.wells('A1')
master_mix_source = cold_rack.wells('A2')
template_sources = template_primer_rack.columns('A')
F_primer_sources = template_primer_rack.columns('B')
R_primer_sources = template_primer_rack.columns('C')

#Define DNA volumes to be added
template_volumes = [template_volume] * num_templates
num_pcr_samples = len(template_volumes)
water_volume = total_PCR_volume - master_mix_volume - (2*primer_volume) - template_volume

mix_trough_bottom = [well.bottom() for well in mix_trough.wells('A1', length=num_templates)]

p200.pick_up_tip()

#Add water
p200.distribute(
	water_volume*(num_replicates+1), water_source, mix_trough_bottom, new_tip='never',
	blow_out=True, disposal_volume=10)

p200.drop_tip()

#distribute MasterMix
p200.distribute(
	master_mix_volume*(num_replicates+1), master_mix_source, mix_trough_bottom, blow_out=True,
	disposal_volume=10)

#distribute F primers
for i in range(len(template_volumes)):
	p200.transfer(
		primer_volume*(num_replicates+1), F_primer_sources(i), mix_trough.wells(i).bottom(),
		mix_after=(2, 20), blow_out=True, new_tip='always')

#distribute R primers
for i in range(len(template_volumes)):
	p200.transfer(
		primer_volume*(num_replicates+1), R_primer_sources(i), mix_trough.wells(i).bottom(),
		mix_after=(2, 20), blow_out=True, touch_tip=True, new_tip='always')

#distribute template DNA
for i in range(len(template_volumes)):
	p20.transfer(
		template_volume*(num_replicates+1), template_sources(i), mix_trough.wells(i).bottom(),
		mix_after=(2, 10), blow_out=True, touch_tip=True, new_tip='always')

#distribute mixes to PCR plate
for i in range(len(template_volumes)):
	pcr_plate_bottom = [well.bottom() for well in pcr_plate.wells(i*num_replicates,
	length=num_replicates, skip=8)]
	p200.distribute(
		total_PCR_volume, mix_trough(i), pcr_plate_bottom, mix_before=(5,200), disposal_volume=5,
		dispense_speed=300)