# coding:utf8
__author__ = 'Marcelo Ferreira da Costa Gomes'

# Apply filters of interest in a dataframe containing SINAN-SRAG data
import pandas as pd
import numpy as np
import argparse
import logging
import re
from argparse import RawDescriptionHelpFormatter
from .insert_epiweek import insert_epiweek
from .delay_table import extract_quantile, delayimputation, createtable

module_logger = logging.getLogger('update_system.sinan_filter_of_interest')
tabela_siglauf = {'RO': 11,
                 'AC': 12,
                 'AM': 13,
                 'RR': 14,
                 'PA': 15,
                 'AP': 16,
                 'TO': 17,
                 'MA': 21,
                 'PI': 22,
                 'CE': 23,
                 'RN': 24,
                 'PB': 25,
                 'PE': 26,
                 'AL': 27,
                 'SE': 28,
                 'BA': 29,
                 'MG': 31,
                 'ES': 32,
                 'RJ': 33,
                 'SP': 35,
                 'PR': 41,
                 'SC': 42,
                 'RS': 43,
                 'MS': 50,
                 'MT': 51,
                 'GO': 52,
                 'DF': 53,
                 'NI': 99}

filtro_dict = {1: '_hospdeath',
               2: '_sragnofever',
               3: ''}

dict_dtypes = {
    'NU_NOTIFIC': 'str',
    'DT_NOTIFIC': 'str',
    'SEM_NOT': 'Int64',
    'ANO': 'Int16',
    'DT_SIN_PRI': 'str',
    'SEM_PRI': 'Int64',
    'SG_UF_NOT': 'str',
    'ID_REGIONA': 'str',
    'CO_REGIONA': 'Int32',
    'ID_MUNICIP': 'str',
    'CO_MUN_NOT': 'Int32',
    'ID_UNIDADE': 'str',
    'CO_UNI_NOT': 'Int32',
    'NM_UN_INTE': 'str',
    'CO_UN_INTE': 'Int32',
    'LAB_AN': 'str',
    'CO_LAB_AN': 'Int32',
    'RES_CAPITAL': 'str',
    'NOT_CAPITAL': 'str',
    'NU_CPF': 'Int64',
    'NU_CNS': 'str',
    'NM_PACIENT': 'str',
    'CS_SEXO': 'str',
    'DT_NASC': 'str',
    'NU_IDADE_N': 'str',
    'TP_IDADE': 'Int8',
    'COD_IDADE': 'str',
    'CS_GESTANT': 'Int8',
    'CS_RACA': 'Int8',
    'CS_ETINIA': 'str',
    'TP_POV_CT': 'str',
    'CS_ESCOL_N': 'Int8',
    'NM_MAE_PAC': 'str',
    'NU_CEP': 'str',
    'ID_PAIS': 'str',
    'CO_PAIS': 'Int16',
    'SG_UF': 'str',
    'ID_RG_RESI': 'str',
    'CO_RG_RESI': 'Int16',
    'ID_MN_RESI': 'str',
    'CO_MUN_RES': 'Int32',
    'NM_BAIRRO': 'str',
    'NM_LOGRADO': 'str',
    'NU_NUMERO': 'str',
    'NM_COMPLEM': 'str',
    'NU_DDD_TEL': 'Int8',
    'NU_TELEFON': 'str',
    'CS_ZONA': 'Int8',
    'SURTO_SG': 'Int8',
    'NOSOCOMIAL': 'Int8',
    'AVE_SUINO': 'Int8',
    'FEBRE': 'Int8',
    'TOSSE': 'Int8',
    'GARGANTA': 'Int8',
    'DISPNEIA': 'Int8',
    'DESC_RESP': 'Int8',
    'SATURACAO': 'Int8',
    'DIARREIA': 'Int8',
    'VOMITO': 'Int8',
    'OUTRO_SIN': 'Int8',
    'OUTRO_DES': 'str',
    'PUERPERA': 'Int8',
    'FATOR_RISC': 'str',
    'CARDIOPATI': 'Int8',
    'HEMATOLOGI': 'Int8',
    'SIND_DOWN': 'Int8',
    'HEPATICA': 'Int8',
    'ASMA': 'Int8',
    'DIABETES': 'Int8',
    'NEUROLOGIC': 'Int8',
    'PNEUMOPATI': 'Int8',
    'IMUNODEPRE': 'Int8',
    'RENAL': 'Int8',
    'OBESIDADE': 'Int8',
    'OBES_IMC': 'str',
    'OUT_MORBI': 'Int8',
    'MORB_DESC': 'str',
    'VACINA': 'Int8',
    'DT_UT_DOSE': 'str',
    'MAE_VAC': 'Int8',
    'DT_VAC_MAE': 'str',
    'M_AMAMENTA': 'Int8',
    'DT_DOSEUNI': 'str',
    'DT_1_DOSE': 'str',
    'DT_2_DOSE': 'str',
    'ANTIVIRAL': 'Int8',
    'TP_ANTIVIR': 'Int8',
    'OUT_ANTIV': 'str',
    'DT_ANTIVIR': 'str',
    'HOSPITAL': 'Int8',
    'DT_INTERNA': 'str',
    'SG_UF_INTE': 'str',
    'ID_RG_INTE': 'str',
    'CO_RG_INTE': 'Int16',
    'ID_MN_INTE': 'str',
    'CO_MU_INTE': 'Int32',
    'UTI': 'Int8',
    'DT_ENTUTI': 'str',
    'DT_SAIDUTI': 'str',
    'SUPORT_VEN': 'Int8',
    'RAIOX_RES': 'Int8',
    'RAIOX_OUT': 'str',
    'DT_RAIOX': 'str',
    'AMOSTRA': 'Int8',
    'DT_COLETA': 'str',
    'TP_AMOSTRA': 'Int8',
    'OUT_AMOST': 'str',
    'REQUI_GAL': 'Int64',
    'IF_RESUL': 'Int8',
    'DT_IF': 'str',
    'POS_IF_FLU': 'Int8',
    'TP_FLU_IF': 'Int8',
    'POS_IF_OUT': 'Int8',
    'IF_VSR': 'Int8',
    'IF_PARA1': 'Int8',
    'IF_PARA2': 'Int8',
    'IF_PARA3': 'Int8',
    'IF_ADENO': 'Int8',
    'IF_OUTRO': 'Int8',
    'DS_IF_OUT': 'object',
    'LAB_IF': 'str',
    'CO_LAB_IF': 'Int64',
    'PCR_RESUL': 'Int8',
    'DT_PCR': 'str',
    'POS_PCRFLU': 'Int8',
    'TP_FLU_PCR': 'Int8',
    'PCR_FLUASU': 'Int8',
    'FLUASU_OUT': 'str',
    'PCR_FLUBLI': 'Int8',
    'FLUBLI_OUT': 'str',
    'POS_PCROUT': 'Int8',
    'PCR_VSR': 'Int8',
    'PCR_PARA1': 'Int8',
    'PCR_PARA2': 'Int8',
    'PCR_PARA3': 'Int8',
    'PCR_PARA4': 'Int8',
    'PCR_ADENO': 'Int8',
    'PCR_METAP': 'Int8',
    'PCR_BOCA': 'Int8',
    'PCR_RINO': 'Int8',
    'PCR_OUTRO': 'Int8',
    'DS_PCR_OUT': 'str',
    'LAB_PCR': 'str',
    'CO_LAB_PCR': 'Int64',
    'CLASSI_FIN': 'Int8',
    'CLASSI_OUT': 'str',
    'CRITERIO': 'Int8',
    'EVOLUCAO': 'Int8',
    'DT_EVOLUCA': 'str',
    'NU_DO': 'Int64',
    'DT_ENCERRA': 'str',
    'OBSERVA': 'str',
    'NOME_PROF': 'str',
    'REG_PROF': 'str',
    'DT_DIGITA': 'str',
    'HISTO_VGM': 'Int8',
    'PAIS_VGM': 'str',
    'CO_PS_VGM': 'Int16',
    'LO_PS_VGM': 'str',
    'DT_VGM': 'str',
    'DT_RT_VGM': 'str',
    'PCR_SARS2': 'Int8',
    'PAC_COCBO': 'str',
    'PAC_DSCBO': 'str',
    'OUT_ANIM': 'str',
    'DOR_ABD': 'Int8',
    'FADIGA': 'Int8',
    'PERD_OLFT': 'Int8',
    'PERD_PALA': 'Int8',
    'TOMO_RES': 'Int8',
    'TOMO_OUT': 'str',
    'DT_TOMO': 'str',
    'TP_TES_AN': 'Int8',
    'DT_RES_AN': 'str',
    'RES_AN': 'Int8',
    'POS_AN_FLU': 'Int8',
    'TP_FLU_AN': 'Int8',
    'POS_AN_OUT': 'Int8',
    'AN_SARS2': 'Int8',
    'AN_VSR': 'Int8',
    'AN_PARA1': 'Int8',
    'AN_PARA2': 'Int8',
    'AN_PARA3': 'Int8',
    'AN_ADENO': 'Int8',
    'AN_OUTRO': 'Int8',
    'DS_AN_OUT': 'str',
    'TP_AM_SOR': 'Int8',
    'SOR_OUT': 'str',
    'DT_CO_SOR': 'str',
    'TP_SOR': 'Int8',
    'OUT_SOR': 'str',
    'DT_RES': 'str',
    'RES_IGG': 'Int8',
    'RES_IGM': 'Int8',
    'RES_IGA': 'Int8',
    'DT_ATUALIZACAO': 'str',
    'DOSE_1_COV': 'str',
    'DOSE_2_COV': 'str',
    'LOTE_1_COV': 'str',
    'LOTE_2_COV': 'str',
    'LAB_PR_COV': 'str',
    'VACINA_COV': 'Int8',
    '@VERSION': 'Int64',
    '@TIMESTAMP': 'str'
}

def eq_notna(s: pd.Series, x):
    return pd.notna(s) & s.eq(x)

def ne_orna(s: pd.Series, x):
    return pd.isna(s) | s.ne(x)

def clean_open_field(s: pd.Series):

    specialchar = '\@|\$|\%|\&|\*'
    s = s.where(~s.str.contains(specialchar, regex=True, na=False), 'CENSURADO')

    return s

def check_covid19_regex(df: pd.DataFrame, column: str):

    if sum(~pd.isnull(df[column])) > 0:
        sars_cov_2_regex = 'novo corona|covid|coovid|cov-2|covd-19|' \
                           'cov2|cov 2|cov- 2|cov -2|cov - 2|c0v-2|cav-2|cav 2|arvs-cov|ars- cov|ars cov|ars - ' \
                           'cov|ars-cov|ars2cov|covi-19|civid 19|cobid-19|covis|cov-2|civid-19'
        sars_cov_2_regex_discard = 'nao detectado para covid19'
        row_filter = (df[column].str.contains(sars_cov_2_regex.upper(), regex=True, na=False)) & \
                     (~df[column].str.contains(sars_cov_2_regex_discard.upper(), regex=True, na=False))
    else:
        row_filter = ~pd.isnull(df[column])

    return row_filter

def readtable(fname, sep=','):
    try:
        df = pd.read_csv(fname, sep=sep, dtype=dict_dtypes, low_memory=False, encoding='utf-8')
    except:
        df = pd.read_csv(fname, sep=sep, dtype=dict_dtypes, low_memory=False, encoding='utf-16')

    return df

def date_cleanup(df, dt_cols):
    '''
    Standardize column data and discard those without notification and/or first symptoms date.
    :param df: notifications data frame
    :param dt_cols: list of data columns
    :return: df: standardized data frame
    '''

    # Filter by notification date
    df = df.where(df != -1, None)
    df[dt_cols] = df[dt_cols].where(df[dt_cols] != 10101, None)
    if df.CRITERIO.dtype == 'O':
        df.CRITERIO = df.CRITERIO.where(df.CRITERIO != 'NÃ', None)
    df.dropna(subset=["DT_SIN_PRI", "DT_NOTIFIC"], inplace=True)

    # Convert all date related columns to datetime format
    for col in dt_cols:
        # Convert all date columns to datetime format. Output will have the format YYYY-MM-DD
        dtformat = '%Y%m%d'
        if sum(~pd.isnull(df[col])) > 0:
            sample = df.loc[~pd.isnull(df[col]), col].values[0]
            if isinstance(sample, str):
                if 'T' in sample:
                    df[col] = pd.to_datetime(df[col].str[:10], errors='coerce', format='%Y-%m-%d')
                else:
                    dtsep = '-'
                    if '/' in sample:
                        dtsep = '/'
                    dttest = pd.DataFrame(list(
                        df.loc[~pd.isnull(df[col]), col].str.split(dtsep)
                    ))
                    maxvals = [int(dttest[i].max()) for i in range(3)]
                    del dttest
                    yearpos = maxvals.index(max(maxvals))
                    if yearpos == 2:
                        if maxvals[1] > 12:
                            dtformat = '%m' + dtsep + '%d' + dtsep + '%Y'
                        else:
                            dtformat = '%d' + dtsep + '%m' + dtsep + '%Y'
                    else:
                        dtformat = '%Y' + dtsep + '%m' + dtsep + '%d'
                    df[col] = pd.to_datetime(df[col], errors='coerce', format=dtformat)
            else:
                df[col] = pd.to_datetime(df[col], errors='coerce', format=dtformat)
        else:
            df[col] = pd.to_datetime(df[col])
    # Discard those neither hospitalized nor deceased. For cases from 2009, keep all:
    df = df[(df.DT_SIN_PRI.apply(lambda x: x.year) == 2009) | (eq_notna(df.HOSPITAL, 1)) | (eq_notna(df.EVOLUCAO, 2))]

    return df

def table_compatibility(df):
    '''
    Create column conversion to make old and new data compatible
    :param df: input data frame
    :return:
    '''

    # Create equivalence between 2020-07-27 new variables and old ones:
    df['DT_IF'] = df['DT_RES_AN'].copy()
    if 'RES_AN' in df.columns:
        df['IF_RESUL'] = df['RES_AN'].copy()

    l_new = ['CO_LAB_AN',
             'POS_AN_FLU',
             'TP_FLU_AN',
             'POS_AN_OUT',
             'AN_VSR',
             'AN_PARA1',
             'AN_PARA2',
             'AN_PARA3',
             'AN_ADENO',
             'AN_OUTRO',
             'DS_AN_OUT']
    cols = df.columns
    col_new = []
    col_old = []
    l_intersec = set(l_new).intersection(cols)
    for li in l_intersec:
        col_new.append(li.replace('AN', 'IF'))
        col_old.append(li)

    if len(l_intersec) > 0:
        df[col_new] = df[col_old].copy()

    return

def symptoms_filter(df):

    # Filter by symptoms:
    # All cases, regardless of year, must either attend symptoms definition or have evolved to
    # death. Besides that, cases from 2009 do not need hospitalization to be considered as true case, per Ministry
    # of Health request. For all other years, they must have been hospitalized or have evolved to death.
    # The filter regarding hospitalization and case evolution is done after date columns consolidation to attend
    # 2009's particularity.

    # all
    df['filtro'] = 0
    # hospdeath:
    df.loc[(eq_notna(df.EVOLUCAO, 2)) | (eq_notna(df.HOSPITAL, 1)), 'filtro'] = 1
    # sragnofever:
    df.loc[(df.filtro == 1) & (
            ((eq_notna(df.TOSSE, 1)) | (eq_notna(df.GARGANTA, 1))) &
            ((eq_notna(df.DISPNEIA, 1)) | (eq_notna(df.SATURACAO, 1)) | (eq_notna(df.DESC_RESP, 1)))
    ), 'filtro'] = 2
    # srag:
    df.loc[(df.filtro == 2) & (eq_notna(df.FEBRE, 1)), 'filtro'] = 3

    return df

def filter_db2019(df, tag=None, filtertype='srag'):
    tgtcols = ['AMOSTRA', 'ANTIVIRAL', 'AVE_SUINO', 'CLASSI_FIN', 'CLASSI_OUT', 'CO_LAB_IF', 'CO_LAB_PCR', 'CO_MU_INTE',
               'CO_MUN_NOT', 'CO_MUN_RES', 'CO_PAIS', 'CO_REGIONA', 'CO_RG_RESI', 'CO_UNI_NOT', 'CRITERIO', 'CS_ETINIA',
               'CO_UN_INTE',
               'CS_RACA', 'CS_SEXO', 'DS_IF_OUT', 'DS_PCR_OUT', 'DT_ANTIVIR',
               'DT_COLETA', 'DT_DIGITA', 'DT_ENCERRA', 'DT_EVOLUCA', 'DT_IF', 'DT_INTERNA', 'DT_NOTIFIC', 'DT_PCR',
               'DT_IFI',
               'DT_SIN_PRI', 'DT_UT_DOSE', 'EVOLUCAO', 'FLUASU_OUT', 'FLUBLI_OUT', 'HOSPITAL',
               'ID_MN_RESI', 'ID_MUNICIP', 'ID_REGIONA', 'ID_RG_RESI', 'IF_ADENO', 'IF_OUTRO', 'IF_PARA1',
               'IF_PARA2', 'IF_PARA3', 'IF_RESUL', 'IF_VSR', 'NU_CEP', 'NU_IDADE_N', 'NU_NOTIFIC', 'NM_BAIRRO',
               'NM_LOGRADO', 'OUT_ANTIVIR', 'PCR_ADENO', 'PCR_BOCA', 'PCR_FLUASU', 'PCR_FLUBLI', 'PCR_METAP',
               'PCR_OUTRO', 'PCR_PARA1',
               'PCR_PARA2', 'PCR_PARA3', 'PCR_PARA4', 'PCR_RESUL', 'PCR_RINO', 'PCR_SARS2', 'PCR_VSR', 'POS_IF_FLU',
               'POS_IF_OUT', 'POS_PCRFLU', 'POS_PCROUT', 'SEM_NOT', 'SEM_PRI', 'SG_UF', 'SG_UF_NOT',
               'TP_ANTIVIR', 'TP_FLU_IF', 'TP_FLU_PCR', 'TP_IDADE', 'VACINA']

    tgt_cols_sintomas = ['DESC_RESP',
                         'DISPNEIA',
                         'DIARREIA',
                         'FEBRE',
                         'GARGANTA',
                         'TOSSE',
                         'SATURACAO',
                         'DIARREIA',
                         'VOMITO',
                         'DOR_ABD',
                         'FADIGA',
                         'PERD_OLFT',
                         'PERD_PALA',
                         'OUTRO_SIN',
                         'OUTRO_DES']

    tgtcols_uti = ['UTI',
                   'DT_ENTUTI',
                   'DT_SAIDUTI',
                   'SUPORT_VEN']

    tgtcols_comorb = ['CS_GESTANT',
                      'FATOR_RISC',
                      'PUERPERA',
                      'CARDIOPATI',
                      'HEMATOLOGI',
                      'HEPATICA',
                      'SIND_DOWN',
                      'HEPATICA',
                      'ASMA',
                      'DIABETES',
                      'NEUROLOGIC',
                      'PNEUMOPATI',
                      'IMUNODEPRE',
                      'RENAL',
                      'OBESIDADE',
                      'OBES_IMC',
                      'OUT_MORBI',
                      'MORB_DESC']

    tgtcols_2020_07_27 = ['TP_TES_AN',
                          'DT_RES_AN',
                          'RES_IGA',
                          'RES_IGG',
                          'RES_IGM',
                          'LAB_AN',
                          'CO_LAB_AN',
                          'POS_AN_FLU',
                          'TP_FLU_AN',
                          'POS_AN_OUT',
                          'AN_SARS2',
                          'AN_VSR',
                          'AN_PARA1',
                          'AN_PARA2',
                          'AN_PARA3',
                          'AN_ADENO',
                          'AN_OUTRO',
                          'DS_AN_OUT',
                          'SUPORT_VEN',
                          'RAIOX_RES',
                          'RAIOX_OUT',
                          'DT_RAIOX',
                          'TOMO_RES',
                          'TOMO_OUT',
                          'DT_TOMO']
    tgt_cols_vac_covid = ['VACINA_COV', 'DOSE_1_COV', 'DOSE_2_COV', 'LAB_PR_COV', 'FNT_IN_COV']

    if 'DT_RES_AN' in df.columns:
        table_compatibility(df)

    tgtcols = list(set(tgtcols).union(tgt_cols_sintomas).union(tgtcols_uti).union(tgtcols_comorb).union(
        tgtcols_2020_07_27).union(tgt_cols_vac_covid))
    if 'DT_IF' in df.columns:
        df['DT_IFI'] = df.DT_IF

    cols = df.columns
    for col in set(tgtcols).difference(cols):
        df[col] = None

    df = df[tgtcols].copy()
    df = symptoms_filter(df)

    regexp = re.compile('^DT')
    dt_cols = list(filter(regexp.match, tgtcols))
    df = date_cleanup(df, dt_cols)

    # Registro da idade, em anos.
    # Campo TP_IDADE indica a escala da variável NU_IDADE_N, sendo:
    # 1-Dias, 2-Meses, 3-Anos
    df['idade_em_anos'] = df.NU_IDADE_N.where(df.TP_IDADE == 3, 0)

    # If sample collection field is empty, convert to unknown:
    df.AMOSTRA = df.AMOSTRA.where(pd.notnull(df.AMOSTRA), 9).astype(int)

    # Convert UF abbreviation to numeric code:
    df.loc[pd.notnull(df.SG_UF_NOT), 'SG_UF_NOT'] = df.loc[pd.notnull(df.SG_UF_NOT), 'SG_UF_NOT'].map(tabela_siglauf)
    df.loc[pd.isnull(df.SG_UF_NOT), 'SG_UF_NOT'] = 99
    df.loc[pd.notnull(df.SG_UF), 'SG_UF'] = df.loc[pd.notnull(df.SG_UF), 'SG_UF'].map(tabela_siglauf)
    df.loc[pd.isnull(df.SG_UF), 'SG_UF'] = 99

    # Clean up PCR_RESUL and IF_RESUL fields:
    def labresultcleanup(dfin, x):
        # If sample was collected, IF/PCR result cannot be unknown or empty:
        dfin.loc[(eq_notna(dfin.AMOSTRA, 1)) & (~dfin[x].isin([1, 2, 3, 4])), x] = 5
        # IF PCR/IF result field is marked unknown but sample was not collected, convert to not tested:
        dfin.loc[(eq_notna(dfin[x], 9)) & (eq_notna(dfin.AMOSTRA, 2)), x] = 4
        # If PCR/IF result field is empty and sample was NOT collected, convert to not tested
        dfin[x] = dfin[x].where(pd.notnull(dfin[x]) | (ne_orna(dfin.AMOSTRA, 2)), 4)
        # If PCR/IF result field is empty and sample field is empty or unknown, convert to unknown
        dfin[x] = dfin[x].where(pd.notnull(dfin[x]) | (dfin.AMOSTRA.isin([1, 2])), 9)

        return

    labresultcleanup(df, 'PCR_RESUL')
    labresultcleanup(df, 'IF_RESUL')

    df['FLU_A'] = pd.Series([], dtype='Int8')
    df['FLU_B'] = pd.Series([], dtype='Int8')
    df['FLU_LAB'] = pd.Series([], dtype='Int8')
    df['VSR'] = pd.Series([], dtype='Int8')
    df['PARA1'] = pd.Series([], dtype='Int8')
    df['PARA2'] = pd.Series([], dtype='Int8')
    df['PARA3'] = pd.Series([], dtype='Int8')
    df['PARA4'] = pd.Series([], dtype='Int8')
    df['ADNO'] = pd.Series([], dtype='Int8')
    df['METAP'] = pd.Series([], dtype='Int8')
    df['BOCA'] = pd.Series([], dtype='Int8')
    df['RINO'] = pd.Series([], dtype='Int8')
    df['SARS2'] = pd.Series([], dtype='Int8')
    df['OTHERS'] = pd.Series([], dtype='Int8')

    df['NEGATIVE'] = pd.Series([], dtype='Int8')
    df['POSITIVE'] = pd.Series([], dtype='Int8')
    df['INCONCLUSIVE'] = pd.Series([], dtype='Int8')
    df['DELAYED'] = pd.Series([], dtype='Int8')
    df['TESTED'] = pd.Series([], dtype='Int8')
    df['NOTTESTED'] = pd.Series([], dtype='Int8')
    df['TESTING_IGNORED'] = pd.Series([], dtype='Int8')

    def labresult(x, y=None, pos=1):
        if all(pd.isnull([x, y])):
            return None
        elif pos in [pd.notna(x), pd.notna(y)]:
            return 1
        else:
            return 0

    mask = (df[['TP_FLU_PCR', 'TP_FLU_IF']].notna().sum(axis=1) >= 1)
    df.loc[mask, 'FLU_A'] = df.loc[mask, 'FLU_A'] = (
            eq_notna(df.TP_FLU_PCR[mask], 1) | eq_notna(df.TP_FLU_IF[mask], 1)
    ).astype('Int8')
    df.loc[mask, 'FLU_B'] = df.loc[mask, 'FLU_A'] = (
            eq_notna(df.TP_FLU_PCR[mask], 2) | eq_notna(df.TP_FLU_IF[mask], 2)
    ).astype('Int8')

    df.loc[(eq_notna(df.POS_IF_FLU, 1)) |
           (eq_notna(df.POS_PCRFLU, 1)) |
           (eq_notna(df.FLU_A, 1)) |
           (eq_notna(df.FLU_B, 1)) |
           ((eq_notna(df.CLASSI_FIN, 1)) & (eq_notna(df.CRITERIO, 1))), 'FLU_LAB'] = 1
    df.loc[(ne_orna(df.FLU_LAB, 1)) & ((eq_notna(df.POS_IF_FLU, 2)) | (eq_notna(df.POS_PCRFLU, 2))), 'FLU_LAB'] = 0
    df.loc[(eq_notna(df.IF_VSR, 1)) | (eq_notna(df.PCR_VSR, 1)), 'VSR'] = 1
    df.loc[(eq_notna(df.IF_PARA1, 1)) | (eq_notna(df.PCR_PARA1, 1)), 'PARA1'] = 1
    df.loc[(eq_notna(df.IF_PARA2, 1)) | (eq_notna(df.PCR_PARA2, 1)), 'PARA2'] = 1
    df.loc[(eq_notna(df.IF_PARA3, 1)) | (eq_notna(df.PCR_PARA3, 1)), 'PARA3'] = 1
    df.loc[eq_notna(df.PCR_PARA4, 1), 'PARA4'] = 1
    df.loc[(eq_notna(df.IF_ADENO, 1)) | (eq_notna(df.PCR_ADENO, 1)), 'ADNO'] = 1
    df.loc[eq_notna(df.PCR_BOCA, 1), 'BOCA'] = 1
    df.loc[eq_notna(df.PCR_RINO, 1), 'RINO'] = 1
    df.loc[eq_notna(df.PCR_METAP, 1), 'METAP'] = 1
    df.loc[(eq_notna(df.IF_OUTRO, 1)) | (eq_notna(df.PCR_OUTRO, 1)), 'OTHERS'] = 1

    mask_covid19_ds_pcr = check_covid19_regex(df, 'DS_PCR_OUT')
    mask_covid19_ds_if = check_covid19_regex(df, 'DS_IF_OUT')

    df.loc[(eq_notna(df.PCR_SARS2, 1)) |
           (eq_notna(df.AN_SARS2, 1)) |
           mask_covid19_ds_pcr |
           mask_covid19_ds_if |
           ((eq_notna(df.CLASSI_FIN, 5)) & (eq_notna(df.CRITERIO, 1))),
           'SARS2'] = 1
    df.loc[(eq_notna(df.PCR_SARS2, 1)) |
           mask_covid19_ds_pcr |
           mask_covid19_ds_if,
           'OTHERS'] = pd.NA

    # Positive cases:
    df.loc[(eq_notna(df.POS_PCRFLU, 1)) | (eq_notna(df.POS_PCROUT, 1)), 'PCR_RESUL'] = 1
    df.loc[(eq_notna(df.POS_IF_FLU, 1)) | (eq_notna(df.POS_IF_OUT, 1)), 'IF_RESUL'] = 1
    df.loc[(eq_notna(df.PCR_RESUL, 1)) |
           (eq_notna(df.IF_RESUL, 1)) |
           (eq_notna(df.FLU_LAB, 1)) |
           (eq_notna(df.SARS2, 1)) |
           (eq_notna(df.VSR, 1)) |
           (eq_notna(df.PARA1, 1)) |
           (eq_notna(df.PARA2, 1)) |
           (eq_notna(df.PARA3, 1)) |
           (eq_notna(df.PARA4, 1)) |
           (eq_notna(df.ADNO, 1)) |
           (eq_notna(df.BOCA, 1)) |
           (eq_notna(df.RINO, 1)) |
           (eq_notna(df.METAP, 1)) |
           (eq_notna(df.OTHERS, 1)),
           'POSITIVE'] = 1

    # Negative cases:
    df.loc[(eq_notna(df.POS_PCRFLU, 2)) & (eq_notna(df.POS_PCROUT, 2)), 'PCR_RESUL'] = 2
    df.loc[(eq_notna(df.POS_IF_FLU, 2)) & (eq_notna(df.POS_IF_OUT, 2)), 'IF_RESUL'] = 2
    mask = (
            (ne_orna(df.POSITIVE, 1)) &
            ((eq_notna(df.FLU_LAB, 0)) | (eq_notna(df.POS_PCROUT, 2)) | (eq_notna(df.POS_IF_OUT, 2)))
    )
    df.loc[mask, 'NEGATIVE'] = 1

    df.loc[(((eq_notna(df.IF_RESUL, 2)) & (df.PCR_RESUL.isin([2, 3, 4, 5, 9]))) |
           ((eq_notna(df.PCR_RESUL, 2)) & (df.IF_RESUL.isin([2, 3, 4, 5, 9])))) &
           (ne_orna(df.POSITIVE, 1)), 'NEGATIVE'] = 1
    df.loc[:, 'INCONCLUSIVE'] = (
                                    (
                                            ((eq_notna(df.IF_RESUL, 3)) & (~df.PCR_RESUL.isin([1, 2, 5]))) |
                                            ((eq_notna(df.PCR_RESUL, 3)) & (~df.IF_RESUL.isin([1, 2, 5])))
                                    ) &
                                    (ne_orna(df.POSITIVE, 1)) &
                                    (ne_orna(df.NEGATIVE, 1))
    ).astype(int)
    df.loc[(eq_notna(df.POSITIVE, 1)) | (eq_notna(df.NEGATIVE, 1)) | (eq_notna(df.INCONCLUSIVE, 1)), 'TESTED'] = 1

    df.loc[:, 'DELAYED'] = (
                               (
                                       ((eq_notna(df.IF_RESUL, 5)) & (pd.isnull(df.TESTED))) |
                                       ((eq_notna(df.PCR_RESUL, 5)) & (pd.isnull(df.TESTED)))
                               ) &
                               (ne_orna(df.POSITIVE, 1)) &
                               (ne_orna(df.NEGATIVE, 1)) &
                               (ne_orna(df.INCONCLUSIVE, 1))
    ).astype(int)

    # Clinical and clinical-epidemiological diagnose:
    df['FLU_CLINIC'] = ((ne_orna(df.POS_IF_FLU, 1)) & (ne_orna(df.POS_PCRFLU, 1)) & (eq_notna(df.CLASSI_FIN, 1)) &
                        (df.CRITERIO.isin([2, 3]))).astype(int)

    notknownrows = (
                           ((eq_notna(df.PCR_RESUL, 9)) | pd.isnull(df.PCR_RESUL)) &
                           ((eq_notna(df.IF_RESUL, 9)) | pd.isnull(df.IF_RESUL)) &
                           (ne_orna(df.POSITIVE, 1)) &
                           (ne_orna(df.NEGATIVE, 1)) &
                           (ne_orna(df.DELAYED, 1)) &
                           (ne_orna(df.INCONCLUSIVE, 1))
    )
    nottestedrows = (
            ~(notknownrows) &
            (df.PCR_RESUL.isin([4, 9])) &
            (df.IF_RESUL.isin([4, 9])) &
            (ne_orna(df.TESTED, 1))
    )
    df['NOTTESTED'] = nottestedrows.astype(int)
    df['TESTING_IGNORED'] = notknownrows.astype(int)

    if tag:
        df['tag'] = tag

    return df

def applysinanfilter(df, tag=None, filtertype='srag'):
    # Filter columns of interest
    # Na solicitação, além das variáveis abaixo, necessitamos também do ID do caso
    tgtcols = ['SEM_NOT', 'DT_NOTIFIC', 'SG_UF_NOT', 'DT_INTERNA', 'DT_SIN_PRI', 'DT_DIGITA', 'HOSPITAL',
               'FEBRE', 'CLASSI_FIN', 'CRITERIO', 'SG_UF', 'ID_MN_RESI', 'ID_RG_RESI', 'SEM_PRI',
               'TOSSE', 'GARGANTA', 'DISPNEIA', 'SATURACAO', 'DESC_RESP', 'EVOLUCAO', 'DT_COLETA', 'IFI', 'DT_IFI',
               'PCR', 'OUT_METODO', 'DS_OUTMET', 'DT_OUTMET', 'RES_FLUA', 'RES_FLUASU', 'RES_FLUB',
               'RES_VSR', 'RES_PARA1', 'RES_PARA2', 'RES_PARA3', 'RES_ADNO', 'RES_OUTRO', 'DT_PCR', 'PCR_RES',
               'PCR_ETIOL', 'PCR_TIPO_H', 'PCR_TIPO_N', 'DT_CULTURA', 'CULT_RES', 'DT_HEMAGLU', 'HEMA_RES',
               'HEMA_ETIOL', 'HEM_TIPO_H', 'HEM_TIPO_N', 'VACINA', 'DT_UT_DOSE', 'ANT_PNEUMO', 'DT_PNEUM',
               'CO_UF_INTE', 'CO_MU_INTE', 'CO_UN_INTE', 'DT_ENCERRA', 'NU_NOTIFIC', 'ID_AGRAVO', 'ID_MUNICIP',
               'ID_REGIONA', 'ID_UNIDADE', 'NU_IDADE_N', 'NM_BAIRRO', 'NM_LOGRADO', 'CS_SEXO', 'CS_RACA',
               'DT_ANTIVIR', 'DT_EVOLUCA']

    tgtcols_uti = ['UTI',
                   'DT_ENTUTI',
                   'DT_SAIDUTI']

    tgtcols_comorb = ['CS_GESTANT',
                      'PUERPERA',
                      'CARDIOPATI',
                      'HEMATOLOGI',
                      'SIND_DOWN',
                      'HEPATICA',
                      'ASMA',
                      'DIABETES',
                      'NEUROLOGIC',
                      'PNEUMOPATI',
                      'IMUNODEPRE',
                      'RENAL',
                      'OBESIDADE',
                      'OBES_IMC',
                      'OUT_MORBI',
                      'MORB_DESC']

    tgtcols = list(set(tgtcols).union(tgtcols_uti).union(tgtcols_comorb))

    cols = df.columns
    if 'RES_VRS' in cols:
        df.rename(columns={'RES_VRS': 'RES_VSR'}, inplace=True)
    if 'DT_PCR_1' in cols:
        df.DT_PCR.update(df.DT_PCR_1)
    if 'DT_OBITO' in cols:
        df.rename(columns={'DT_OBITO': 'DT_EVOLUCA'}, inplace=True)
        if 'DT_EVOL' in cols:
            df.DT_EVOLUCA.update(df.DT_EVOL)
    elif 'DT_EVOL' in cols:
        df.rename(columns={'DT_EVOL': 'DT_EVOLUCA'}, inplace=True)
    if 'METABOLICA' in cols:
        df.rename(columns={'METABOLICA': 'DIABETES'}, inplace=True)

    cols = df.columns
    for col in set(tgtcols).difference(cols):
        df[col] = None

    df = df[tgtcols].copy()

    df = symptoms_filter(df)

    regexp = re.compile('^DT')
    dt_cols = list(filter(regexp.match, tgtcols))
    df = date_cleanup(df, dt_cols)

    # Create columns related to lab result
    # Rows with lab test:
    labrows = ((df.PCR_RES.isin([1, 2, 3])) |
               (df.CULT_RES.isin([1, 2])) |
               (df.HEMA_RES.isin([1, 2, 3])) |
               (eq_notna(df.IFI, 1)) |
               (eq_notna(df.PCR, 1)) |
               (eq_notna(df.OUT_METODO, 1)))

    notknownrows = (
        pd.isnull(df.PCR_RES) &
        pd.isnull(df.CULT_RES) &
        pd.isnull(df.HEMA_RES) &
        pd.isnull(df.IFI) &
        pd.isnull(df.PCR) &
        pd.isnull(df.OUT_METODO)
    )

    nottestedrows = (
        ~(notknownrows) &
        (pd.isnull(df.PCR_RES) | (df.PCR_RES.isin([4]))) &
        (pd.isnull(df.CULT_RES) | (df.CULT_RES.isin([3]))) &
        (pd.isnull(df.HEMA_RES) | (df.HEMA_RES.isin([4]))) &
        (pd.isnull(df.IFI) | (eq_notna(df.IFI, 2))) &
        (pd.isnull(df.PCR) | (eq_notna(df.PCR, 2))) &
        (pd.isnull(df.OUT_METODO) | (eq_notna(df.OUT_METODO, 2)))
    )

    df['FLU_A'] = pd.Series([], dtype='Int8')
    df['FLU_B'] = pd.Series([], dtype='Int8')
    df['FLU_LAB'] = pd.Series([], dtype='Int8')
    df['VSR'] = pd.Series([], dtype='Int8')
    df['PARA1'] = pd.Series([], dtype='Int8')
    df['PARA2'] = pd.Series([], dtype='Int8')
    df['PARA3'] = pd.Series([], dtype='Int8')
    df['PARA4'] = pd.Series([], dtype='Int8')
    df['ADNO'] = pd.Series([], dtype='Int8')
    df['METAP'] = pd.Series([], dtype='Int8')
    df['BOCA'] = pd.Series([], dtype='Int8')
    df['RINO'] = pd.Series([], dtype='Int8')
    df['SARS2'] = pd.Series([], dtype='Int8')
    df['OTHERS'] = pd.Series([], dtype='Int8')

    df['NEGATIVE'] = pd.Series([], dtype='Int8')
    df['POSITIVE'] = pd.Series([], dtype='Int8')
    df['INCONCLUSIVE'] = pd.Series([], dtype='Int8')
    df['DELAYED'] = pd.Series([], dtype='Int8')
    df['TESTED'] = pd.Series([], dtype='Int8')
    df['NOTTESTED'] = pd.Series([], dtype='Int8')
    df['TESTING_IGNORED'] = pd.Series([], dtype='Int8')

    df['TESTED'] = labrows.astype(int)
    df['NOTTESTED'] = nottestedrows.astype(int)
    df['TESTING_IGNORED'] = notknownrows.astype(int)

    df.loc[labrows, 'FLU_A'] = ((df.PCR_ETIOL[labrows].isin([1, 2, 4])) | (df.HEMA_ETIOL[labrows].isin([1, 2, 4])) |
                                (eq_notna(df.RES_FLUA[labrows], 1))).astype(int)
    df.loc[labrows, 'FLU_B'] = ((eq_notna(df.PCR_ETIOL[labrows], 3)) | (eq_notna(df.HEMA_ETIOL[labrows], 3)) |
                                (eq_notna(df.RES_FLUB[labrows], 1))).astype(int)
    df.loc[labrows, 'VSR'] = (eq_notna(df.RES_VSR[labrows], 1)).astype(int)
    df.loc[labrows, 'PARA1'] = (eq_notna(df.RES_PARA1[labrows], 1)).astype(int)
    df.loc[labrows, 'PARA2'] = (eq_notna(df.RES_PARA2[labrows], 1)).astype(int)
    df.loc[labrows, 'PARA3'] = (eq_notna(df.RES_PARA3[labrows], 1)).astype(int)
    df.loc[labrows, 'ADNO'] = (eq_notna(df.RES_ADNO[labrows], 1)).astype(int)
    df.loc[labrows, 'SARS2'] = 0

    df.loc[labrows, 'OTHERS'] = (
        (eq_notna(df.PCR_ETIOL[labrows], 5)) |
        (eq_notna(df.HEMA_ETIOL[labrows], 5)) |
        (eq_notna(df.RES_OUTRO[labrows], 1))).astype(int)
    df.loc[labrows, 'DELAYED'] = ((pd.isnull(df.PCR_RES[labrows]) | eq_notna(df.PCR_RES[labrows], 4)) &
                                  (pd.isnull(df.HEMA_RES[labrows]) | eq_notna(df.HEMA_RES[labrows], 4)) &
                                  (pd.isnull(df.RES_FLUA[labrows]) | eq_notna(df.RES_FLUA[labrows], 4)) &
                                  (pd.isnull(df.RES_FLUB[labrows]) | eq_notna(df.RES_FLUB[labrows], 4)) &
                                  (pd.isnull(df.RES_VSR[labrows]) | eq_notna(df.RES_VSR[labrows], 4)) &
                                  (pd.isnull(df.RES_PARA1[labrows]) | eq_notna(df.RES_PARA1[labrows], 4)) &
                                  (pd.isnull(df.RES_PARA2[labrows]) | eq_notna(df.RES_PARA2[labrows], 4)) &
                                  (pd.isnull(df.RES_PARA3[labrows]) | eq_notna(df.RES_PARA3[labrows], 4)) &
                                  (pd.isnull(df.RES_ADNO[labrows]) | eq_notna(df.RES_ADNO[labrows], 4)) &
                                  (pd.isnull(df.RES_OUTRO[labrows]) | eq_notna(df.RES_OUTRO[labrows], 4))).astype(int)
    df.loc[labrows, 'INCONCLUSIVE'] = ((eq_notna(df.DELAYED[labrows], 0)) &
                                       (pd.isnull(df.PCR_RES[labrows]) | df.PCR_RES[labrows].isin([3, 4])) &
                                       (pd.isnull(df.HEMA_RES[labrows]) | df.HEMA_RES[labrows].isin([3, 4])) &
                                       (pd.isnull(df.RES_FLUA[labrows]) | df.RES_FLUA[labrows].isin([3, 4])) &
                                       (pd.isnull(df.RES_FLUB[labrows]) | df.RES_FLUB[labrows].isin([3, 4])) &
                                       (pd.isnull(df.RES_VSR[labrows]) | df.RES_VSR[labrows].isin([3, 4])) &
                                       (pd.isnull(df.RES_PARA1[labrows]) | df.RES_PARA1[labrows].isin([3, 4])) &
                                       (pd.isnull(df.RES_PARA2[labrows]) | df.RES_PARA2[labrows].isin([3, 4])) &
                                       (pd.isnull(df.RES_PARA3[labrows]) | df.RES_PARA3[labrows].isin([3, 4])) &
                                       (pd.isnull(df.RES_ADNO[labrows]) | df.RES_ADNO[labrows].isin([3, 4])) &
                                       (pd.isnull(df.RES_OUTRO[labrows]) | df.RES_OUTRO[labrows].isin([3, 4]))).astype(
        int)
    df.loc[labrows, 'NEGATIVE'] = ((eq_notna(df.FLU_A[labrows], 0)) &
                                   (eq_notna(df.FLU_B[labrows], 0)) &
                                   (eq_notna(df.VSR[labrows], 0)) &
                                   (eq_notna(df.PARA1[labrows], 0)) &
                                   (eq_notna(df.PARA2[labrows], 0)) &
                                   (eq_notna(df.PARA3[labrows], 0)) &
                                   (eq_notna(df.ADNO[labrows], 0)) &
                                   (eq_notna(df.OTHERS[labrows], 0)) &
                                   (eq_notna(df.DELAYED[labrows], 0)) &
                                   (eq_notna(df.INCONCLUSIVE[labrows], 0))).astype(int)
    df.loc[labrows &
           (ne_orna(df.INCONCLUSIVE, 1)) &
           (ne_orna(df.NEGATIVE, 1)) &
           (ne_orna(df.DELAYED, 1)),
           'POSITIVE'] = 1

    # Clinical and clinical-epidemiological diagnose:
    df['FLU_CLINIC'] = ((ne_orna(df.FLU_A, 1)) &
                        (ne_orna(df.FLU_B, 1)) &
                        (eq_notna(df.CLASSI_FIN, 1)) &
                        (df.CRITERIO.isin([2, 3]))).astype(int)

    df.NU_IDADE_N = df.NU_IDADE_N.astype(np.float)

    def f_idade(x):

        # System registers age with the following format:
        # TAAA
        # T: type (1-hours, 2-days, 3-months, 4-years)
        #   Hours used only if age < 24h, days used only if 24h <= age < 30d, months only if 30d <= a < 12 months
        # AAA: count on the respective scale
        # Ex.:
        # 2010 : 10 days
        # 3010: 10 months
        # 4010: 10 years
        if np.isnan(x):
            return np.nan
        if x < 4000:
            return 0
        else:
            return x - 4000

    if (df.NU_IDADE_N > 4000).any(axis = 0):
        df['idade_em_anos'] = df['NU_IDADE_N'].apply(f_idade)
    else:
        df['idade_em_anos'] = df['NU_IDADE_N']
    if tag:
        df['tag'] = tag

    return df

def delays_dist(df_in=pd.DataFrame(), filtertype='srag', append_file=None, dfappend=None):
    module_logger.info('Entered function: delay_dist')

    if filtertype not in ['srag', 'sragnofever', 'hospdeath']:
        exit('Invalid filter type: %s' % filtertype)

    suff = ''
    if filtertype != 'srag':
        suff = '_%s' % filtertype

    df_reg = pd.read_csv('../data/regioesclimaticas.csv', low_memory=False)[['Código', 'Região', 'Região oficial']]
    df_reg.rename(columns={'Código': 'UF', 'Região': 'Regional', 'Região oficial': 'Regiao'}, inplace=True)

    def _cleanup(df_input):
        tgt_cols = ['SG_UF_NOT',
                    'DT_NOTIFIC_epiyearweek',
                    'DT_NOTIFIC_epiyear',
                    'DT_NOTIFIC_epiweek',
                    'DT_SIN_PRI_epiyearweek',
                    'DT_SIN_PRI_epiyear',
                    'DT_SIN_PRI_epiweek',
                    'DT_DIGITA_epiyearweek',
                    'DT_DIGITA_epiyear',
                    'DT_DIGITA_epiweek',
                    'Notific2Digita_DelayWeeks',
                    'SinPri2Digita_DelayWeeks',
                    'SinPri2Antivir_DelayWeeks',
                    'SinPri2Notific_DelayWeeks',
                    'SinPri2Coleta_DelayWeeks',
                    'SinPri2Interna_DelayWeeks',
                    'Notific2Encerra_DelayWeeks',
                    'Coleta2IFI_DelayWeeks',
                    'Coleta2PCR_DelayWeeks',
                    'Notific2Coleta_DelayWeeks',
                    'Notific2Antivir_DelayWeeks',
                    'Digita2Antivir_DelayWeeks',
                    'Interna2Evoluca_DelayWeeks',

                    'Notific2Digita_DelayDays',
                    'SinPri2Digita_DelayDays',
                    'SinPri2Antivir_DelayDays',
                    'SinPri2Notific_DelayDays',
                    'SinPri2Coleta_DelayDays',
                    'SinPri2Interna_DelayDays',
                    'Notific2Encerra_DelayDays',
                    'Coleta2IFI_DelayDays',
                    'Coleta2PCR_DelayDays',
                    'Notific2Coleta_DelayDays',
                    'Notific2Antivir_DelayDays',
                    'Digita2Antivir_DelayDays',
                    'Interna2Evoluca_DelayDays',

                    'sragflu',
                    'obitoflu',
                    'sragcovid',
                    'obitocovid',
                    'obito',
                    'filtro']
        df = df_input[tgt_cols].rename(columns={'DT_NOTIFIC_epiyear': 'epiyear',
                                                'DT_NOTIFIC_epiweek': 'epiweek',
                                                'SG_UF_NOT': 'UF'})

        df.loc[pd.isnull(df.UF), 'UF'] = 99
        df.UF = df.UF.astype(float).astype(int)
        # Insert region info:
        df = df.merge(df_reg, on='UF')
        df['Pais'] = 'BR'

        cols = ['DT_DIGITA_epiyear', 'DT_DIGITA_epiweek', 'SinPri2Digita_DelayWeeks']
        try:
            df[cols] = df[cols].astype(int)
        except:
            df[cols] = df[cols].astype(float)
        cols = ['epiyear', 'epiweek']
        try:
            df[cols] = df[cols].astype(int)
        except:
            df[cols] = df[cols].astype(float)
        cols = ['DT_SIN_PRI_epiyear', 'DT_SIN_PRI_epiweek']
        try:
            df[cols] = df[cols].astype(int)
        except:
            df[cols] = df[cols].astype(float)
        return df

    df = _cleanup(df_in)

    def wide2long(dfwide2long):

        dfwide2long['id'] = df.index
        dfwide2long['srag'] = 1
        varcols = ['srag',
                   'sragflu',
                   'obitoflu',
                   'sragcovid',
                   'obitocovid',
                   'obito']
        dfwide2long_out = dfwide2long[['id'] + varcols].melt(id_vars='id',
                                                             var_name='dado',
                                                             value_name='valor')[lambda x: (x.valor == 1)]
        dfwide2long_out = dfwide2long_out.merge(dfwide2long, how='left', on='id').drop(columns=['valor', 'id']+varcols)
        dfwide2long.drop(columns=['id', 'srag'], inplace=True)

        return dfwide2long_out

    df_out = wide2long(df)

    out_cols = ['Coleta2IFI_DelayDays', 'Coleta2IFI_DelayWeeks', 'Coleta2PCR_DelayDays',
                'Coleta2PCR_DelayWeeks', 'DT_DIGITA_epiweek', 'DT_DIGITA_epiyear',
                'DT_DIGITA_epiyearweek', 'DT_NOTIFIC_epiyearweek', 'DT_SIN_PRI_epiweek',
                'DT_SIN_PRI_epiyear', 'DT_SIN_PRI_epiyearweek',
                'Digita2Antivir_DelayDays', 'Digita2Antivir_DelayWeeks',
                'Notific2Antivir_DelayDays', 'Notific2Antivir_DelayWeeks',
                'Notific2Coleta_DelayDays', 'Notific2Coleta_DelayWeeks',
                'Notific2Digita_DelayDays', 'Notific2Digita_DelayWeeks',
                'Notific2Encerra_DelayDays', 'Notific2Encerra_DelayWeeks',
                'SinPri2Antivir_DelayDays', 'SinPri2Antivir_DelayWeeks',
                'SinPri2Coleta_DelayDays', 'SinPri2Coleta_DelayWeeks',
                'SinPri2Digita_DelayDays', 'SinPri2Digita_DelayWeeks',
                'SinPri2Notific_DelayDays', 'SinPri2Notific_DelayWeeks',
                'SinPri2Interna_DelayDays', 'SinPri2Interna_DelayWeeks',
                'Interna2Evoluca_DelayDays', 'Interna2Evoluca_DelayWeeks',
                'UF', 'dado', 'filtro',
                'epiweek', 'epiyear', 'Regional', 'Regiao', 'Pais']

    df_out = df_out.sort_values(by=['dado', 'UF', 'DT_SIN_PRI_epiyearweek', 'DT_NOTIFIC_epiyearweek',
                                    'DT_DIGITA_epiyearweek'])
    df_out = df_out.reset_index()[out_cols]
    if append_file:
        tmp = pd.read_csv(append_file)
        df_out = tmp.append(df_out[out_cols], ignore_index=True, sort=False)

    for k, suffix in filtro_dict.items():
        df_out.loc[(df_out.filtro >= k), out_cols].to_csv('../../data/data/delay_table%s.csv' % suffix, index=False)

    #### Opportunities estimation
    tgt_cols = ['UF', 'Regional', 'Regiao', 'Pais', 'dado', 'filtro', 'DT_SIN_PRI_epiyearweek', 'DT_SIN_PRI_epiyear',
                'DT_SIN_PRI_epiweek', 'SinPri2Digita_DelayWeeks', 'DT_DIGITA_epiyearweek', 'DT_DIGITA_epiyear',
                'DT_DIGITA_epiweek']
    # Generate quantiles' file:
    df_out = df_out[tgt_cols].rename(columns={'DT_SIN_PRI_epiyearweek': 'epiyearweek', 'DT_SIN_PRI_epiyear': 'epiyear',
                                     'DT_SIN_PRI_epiweek': 'epiweek', 'SinPri2Digita_DelayWeeks': 'delayweeks'})
    df_out.UF = df_out.UF.astype(int).astype(str)

    for i, ft in enumerate(['hospdeath', 'sragnofever', 'srag']):
        extract_quantile(df_out[df_out.filtro >= (i+1)], ft)

    del df_out

    # Impute digitalization date if needed:
    if dfappend is not None:
        dfappend = _cleanup(dfappend)
        df = dfappend.append(df, ignore_index=True, sort=False)

    opp_cols = list(set(tgt_cols).difference(['dado']).union(['sragflu', 'obitoflu', 'sragcovid', 'obitocovid',
                                                              'obito']))
    df = df[opp_cols].rename(columns={'DT_SIN_PRI_epiyearweek': 'epiyearweek', 'DT_SIN_PRI_epiyear': 'epiyear',
                                      'DT_SIN_PRI_epiweek': 'epiweek', 'SinPri2Digita_DelayWeeks': 'delayweeks'})
    df.UF = df.UF.astype(int).astype(str)
    df = delayimputation(df)

    df_out = wide2long(df)

    # Create tables and save to file:
    for k, suffix in filtro_dict.items():
        df_outnew = createtable(df_out[df_out.filtro >= k])
        for dataset in df_out.dado.unique():
            fout = '../clean_data/%s%s_sinpri2digita_table_weekly.csv' % (dataset, suffix)
            module_logger.info('Write table %s' % fout)
            df_outnew[df_outnew.dado == dataset].to_csv(fout, index=False)

    return

def main(flist, sep=',', yearmax=None, filtertype='srag', append_cases=None, append_delay=None):

    if filtertype not in ['srag', 'sragnofever', 'hospdeath']:
        module_logger.error('Invalid filter type: %s', filtertype)
        exit(1)

    suff = ''
    if filtertype != 'srag':
        suff = '_%s' % filtertype

    df = pd.DataFrame()
    for fname in flist:
        module_logger.info('Processing database file: %s', fname)
        dftmp = readtable(fname, sep)
        # Check if data file has 2019's database or not:
        if int(re.findall(r'\d+', fname)[0]) < 2019:
            module_logger.info('DB pre-2019')
            df = df.append(applysinanfilter(dftmp, tag=fname, filtertype=filtertype), ignore_index=True, sort=True)
        else:
            module_logger.info('DB 2019 onwards')
            df = df.append(filter_db2019(dftmp, tag=fname, filtertype=filtertype), ignore_index=True, sort=True)

    del dftmp

    if yearmax:
        df = df[(df.DT_SIN_PRI.apply(lambda x: x.year) <= yearmax)]

    df.NM_LOGRADO = clean_open_field(df.NM_LOGRADO)
    df.DS_PCR_OUT = clean_open_field(df.DS_PCR_OUT)
    df.DS_IF_OUT = clean_open_field(df.DS_IF_OUT)
    df.DS_AN_OUT = clean_open_field(df.DS_AN_OUT)
    df.OUTRO_DES = clean_open_field(df.OUTRO_DES)
    df.MORB_DESC = clean_open_field(df.MORB_DESC)
    df.RAIOX_OUT = clean_open_field(df.RAIOX_OUT)
    df.TOMO_OUT = clean_open_field(df.TOMO_OUT)

    # Clean obvious duplicates
    df.drop_duplicates(inplace=True, ignore_index=True)

    # Convert date fields to text and leave NaT as empty cell
    regexp = re.compile('^DT')
    target_cols = list(filter(regexp.match, df.columns))
    df[target_cols] = df[target_cols].applymap(lambda x: str(x.date())).where(lambda x: x != 'NaT', np.nan)
    df = insert_epiweek(df)

    if 'ID_UNIDADE' in df.columns:
        df.CO_UNI_NOT.update(df.ID_UNIDADE)

    mask_sanity_check = (
            (eq_notna(df.SARS2, 1)) &
            (
                    (df.DT_SIN_PRI_epiyear.astype(int) < 2020) |
                    ((df.DT_SIN_PRI_epiyear.astype(int) == 2020) &
                     (df.DT_SIN_PRI_epiweek.astype(int) < 8))
            )
    )
    if sum((df.SARS2[mask_sanity_check])) > 1:
        print('ATENÇÃO: Caso de COVID-19 anterior a 2020 08. Entrar em contato com GT-Influenza')
        df.loc[mask_sanity_check, 'SARS2'] = 0

    if append_cases:
        tmp = readtable(append_cases)
        dfout = tmp.append(df, ignore_index=True)
    else:
        dfout = df.copy()

    mask_flu = ((eq_notna(dfout.FLU_A, 1)) |
                (eq_notna(dfout.FLU_B, 1)) |
                (eq_notna(dfout.FLU_CLINIC, 1)))
    mask_covid = (eq_notna(dfout.SARS2, 1))
    mask_obito = (eq_notna(dfout.EVOLUCAO, 2))
    mask_obitoflu = mask_flu & mask_obito
    mask_obitocovid = mask_covid & mask_obito

    for k, suffix in filtro_dict.items():
        dfout[dfout.filtro >= k].to_csv('../clean_data/clean_data_srag%s_epiweek.csv' % suffix, index=False)
        dfout[(dfout.filtro >= k) & mask_flu].to_csv('../clean_data/clean_data_sragflu%s_epiweek.csv' % suffix, index=False)
        dfout[(dfout.filtro >= k) & mask_obitoflu].to_csv('../clean_data/clean_data_obitoflu%s_epiweek.csv' % suffix,
                                                    index=False)
        dfout[(dfout.filtro >= k) & mask_covid].to_csv('../clean_data/clean_data_sragcovid%s_epiweek.csv' % suffix,
                                                 index=False)
        dfout[(dfout.filtro >= k) & mask_obitocovid].to_csv('../clean_data/clean_data_obitocovid%s_epiweek.csv' % suffix,
                                                      index=False)
        dfout[(dfout.filtro >= k) & mask_obito].to_csv('../clean_data/clean_data_obito%s_epiweek.csv' % suffix, index=False)
    del dfout

    def masks(dfin: pd.DataFrame):
        mask_flu = ((eq_notna(dfin.FLU_A, 1)) |
                    (eq_notna(dfin.FLU_B, 1)) |
                    (eq_notna(dfin.FLU_CLINIC, 1)))
        mask_covid = (eq_notna(dfin.SARS2, 1))
        mask_obito = (eq_notna(dfin.EVOLUCAO, 2))
        mask_obitoflu = mask_flu & mask_obito
        mask_obitocovid = mask_covid & mask_obito
        dfin['obito'] = mask_obito.astype(int)
        dfin['sragflu'] = mask_flu.astype(int)
        dfin['obitoflu'] = mask_obitoflu.astype(int)
        dfin['sragcovid'] = mask_covid.astype(int)
        dfin['obitocovid'] = mask_obitocovid.astype(int)

        return

    masks(df)
    if append_cases:
        masks(tmp)
        delays_dist(df, filtertype, append_delay, dfappend=tmp)
    else:
        delays_dist(df, filtertype)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Clean SINAN SRAG table.\n" +
                                                 "python3 sinan_clean.py --path ../data/influ*.csv --sep ,\n",
                                     formatter_class=RawDescriptionHelpFormatter)
    parser.add_argument('--path', nargs='*', action='append', help='Path to data file')
    parser.add_argument('--sep', help='Column separator', default=',')
    parser.add_argument('--year', help='Maximum year', default=None)
    parser.add_argument('--filtertype', help='Default=srag. Which filter should be used? [srag, sragnofever, '
                                             'hospdeath]', default='srag')
    args = parser.parse_args()
    print(args)
    main(args.path[0], args.sep, args.year, args.filtertype)


from fastapi import FastAPI, HTTPException
from pydantic import BaseModel

class Line(BaseModel):
    matter: str
    teacher: str
    coef: float
    ects: float
    cc1: float
    cc2: float
    exam: float

app = FastAPI()

@app.get("/grades")
def grades():
    with open("exportFIles/note.txt", "r", encoding="utf-8") as file:
        return file.readlines()

@app.get("/grades/{matter}")
def grades_by_matter(matter):
    with open("exportFIles/note.txt", "r", encoding="utf-8") as file:
        for line in file.readlines():
            if matter in line:
                values = line[line.find(".00") + 9:line.find("\n")]
                if values == "":
                    raise HTTPException(status_code=404, detail="No grades in this matter.")

                average = calculate_average(values)
                values = values.split()
                values = [v.replace(',', '.') for v in values]
                vals_float = [float(v) for v in values]
                return {"grades": vals_float, "average": average}
        raise HTTPException(status_code=404, detail="This matter doesn't exist.")

def calculate_average(values: str) -> float:
    vals = values.split()
    vals = [v.replace(',', '.') for v in vals]
    vals_float = [float(v) for v in vals]
    average = sum(vals_float) / len(vals_float)
    average_rounded = round(average, 2)
    return average_rounded

@app.get("/grades/semester/{semester}")
def gradesBySemester(semester):
    with open("exportFIles/note.txt", "r") as file:
        matter_values = 0
        count = 0
        for line in file.readlines():
            if semester in line:
                coef_str = line[line.find(".00") - 1:line.find(".00") + 3]
                if coef_str != "":
                    coef = float(coef_str.split()[0]) if coef_str.split()[0].isdigit() else 0.0

                    values = line[line.find(".00") + 9:line.find("\n")]
                    if values == "":
                        raise HTTPException(status_code=404, detail="No grades in this semester.")

                    average = calculate_average(values)
                    for _ in range(int(coef)):
                        matter_values += average
                        count += 1
        return {"average": round(matter_values / count, 2) if count != 0 else 0.0}

@app.get("/students")
def students():
    with open("exportFIles/trobiEleve3al.txt", "r") as file:
        return file.readlines()

@app.get("/teachers")
def teachers():
    with open("exportFIles/trobiTeachers.txt", "r") as file:
        return file.readlines()

@app.get("/planning")
def planning():
    with open("exportFIles/Planing.txt", "r") as file:
        return file.readlines()


from tokenizers import BertWordPieceTokenizer
import numpy as np
import argparse
import torch

parser = argparse.ArgumentParser(description="Numerize text into numpy format")
parser.add_argument('--vocab', default='path/to/vocab.txt',
                    help="path to vocabulary file")
parser.add_argument('--merge', default='',
                    help="path to merge file")
parser.add_argument('--input', default='en.train',
                    help="path to tokenized text file")
parser.add_argument('--bin_path', default='en.train.pth',
                    help="path to binary file output")
parser.add_argument("--model", choices=["bert", "roberta"], default="bert",
                    help="source pre-trained model")
params = parser.parse_args()

if params.model == 'bert':
    CLS_TOKEN, CLS_INDEX = "[CLS]", 101
    SEP_TOKEN, SEP_INDEX = "[SEP]", 102
    UNK_TOKEN, UNK_INDEX = "[UNK]", 100
    PAD_TOKEN, PAD_INDEX = "[PAD]", 0
    MASK_TOKEN, MASK_INDEX = "[MASK]", 103
elif params.model == 'roberta':
    CLS_TOKEN, CLS_INDEX = '<s>', 0
    SEP_TOKEN, SEP_INDEX = '</s>', 2
    UNK_TOKEN, UNK_INDEX = '<unk>', 3
    PAD_TOKEN, PAD_INDEX = '<pad>', 1
    MASK_TOKEN, MASK_INDEX = '<mask>', 50264

def numerize(vocab_path, input_path, bin_path):
    tokenizer = BertWordPieceTokenizer(
        vocab_path,
        unk_token=UNK_TOKEN,
        sep_token=SEP_TOKEN,
        cls_token=CLS_TOKEN,
        pad_token=PAD_TOKEN,
        mask_token=MASK_TOKEN,
        lowercase=False,
        strip_accents=False)
    sentences = []
    with open(input_path, 'r') as f:
        batch_stream = []
        for i, line in enumerate(f):
            batch_stream.append(line)
            if i % 1000 == 0:
                res = tokenizer.encode_batch(batch_stream)
                batch_stream = []
                # flatten the list
                for s in res:
                    sentences.extend(s.ids[1:])
            if i % 100000 == 0:
                print(f'processed {i} lines')

    print('convert the data to numpy')

    # convert data to numpy format in uint16
    if tokenizer.get_vocab_size() < 1 << 16:
        sentences = np.uint16(sentences)
    else:
        assert tokenizer.get_vocab_size() < 1 << 31
        sentences = np.int32(sentences)

    # save special tokens for later processing
    sep_index = tokenizer.token_to_id(SEP_TOKEN)
    cls_index = tokenizer.token_to_id(CLS_TOKEN)
    unk_index = tokenizer.token_to_id(UNK_TOKEN)
    mask_index = tokenizer.token_to_id(MASK_TOKEN)
    pad_index = tokenizer.token_to_id(PAD_TOKEN)

    # sanity check
    assert sep_index == SEP_INDEX
    assert cls_index == CLS_INDEX
    assert unk_index == UNK_INDEX
    assert pad_index == PAD_INDEX
    assert mask_index == MASK_INDEX

    print('collect statistics')
    # collect some statistics of the dataset
    n_unks = (sentences == unk_index).sum()
    n_toks = len(sentences)
    p_unks = n_unks * 100. / n_toks
    n_seqs = (sentences == sep_index).sum()
    print(f'| {n_seqs} sentences - {n_toks} tokens - {p_unks:.2f}% unknown words')

    # print some statistics
    data = {'sentences': sentences,
            'sep_index': sep_index,
            'cls_index': cls_index,
            'unk_index': unk_index,
            'pad_index': pad_index,
            'mask_index': mask_index}

    torch.save(data, bin_path, pickle_protocol=4)

numerize(params.vocab, params.input, params.bin_path)


import random
import string
import cv2
import os
import argparse

BASE_DIR = os.getcwd()

def cut_img(image, column_num, row_num, prefix_name):
    """
    image: 이미지 경로
    column_num: 가로로 자를 갯수
    row_num: 세로로 자를 갯수
    prefix_name: 고정된 이름
    """
    if os.path.isdir(os.path.join(BASE_DIR, 'cuted_img')):
        for file in os.scandir(os.path.join(BASE_DIR, 'cuted_img')):
            os.remove(file.path)
    else:
        os.mkdir(os.path.join(BASE_DIR, 'cuted_img'))

    h, w, _ = image.shape
    if h % row_num != 0 or w % column_num != 0:
        image = image[:h-(h % row_num), :w-(w % column_num)]

    for i in range(row_num):
        for j in range(column_num):
            cut_img = image[i*int(h//row_num):(i+1)*int(h//row_num), j*int(w//column_num):(j+1)*int(w//column_num)]

            # random 수정
            rand = random.uniform(0,1)
            if rand > 0.5:
                cut_img = cv2.flip(cut_img,1) # 좌우 반전

            rand = random.uniform(0,1)
            if rand > 0.5:
                cut_img = cv2.flip(cut_img,0) # 상하 반전

            rand = random.uniform(0,1)
            if rand > 0.5:
                cut_img = cv2.rotate(cut_img, cv2.ROTATE_90_COUNTERCLOCKWISE)

            random_name_len = 10
            random_name = ""
            for _ in range(random_name_len):
                random_name += str(random.choice(string.ascii_lowercase))

            cv2.imwrite(f'cuted_img/{prefix_name}-{random_name}.png', cut_img)

if __name__=="__main__":
    parser = argparse.ArgumentParser(description="Divide image into patches")

    parser.add_argument("image_file_name", type=str)
    parser.add_argument("column_num", type=int)
    parser.add_argument("row_num", type=int)
    parser.add_argument("prefix_output_filename", type=str)

    configs = parser.parse_args()

    img = cv2.imread(configs.image_file_name, cv2.IMREAD_COLOR)
    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)

    cut_img(img, configs.column_num, configs.row_num, configs.prefix_output_filename)


# -*- coding: utf-8 -*-
"""
Created on Tue Jul 14 20:21:07 2020

@author: gabriel bustamante
"""

import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt

def get_attributes_list(information_level, attributes_description):

    attributes_list = list(attributes_description[attributes_description['information'] == information_level]['attribute'].unique())

    if information_level == '125m x 125m Grid':
        attributes_list += ['D19_BUCH_CD', 'D19_LETZTER_KAUF_BRANCHE', 'D19_LOTTO', 'D19_SOZIALES']
    if information_level == 'Building':
        attributes_list += ['FIRMENDICHTE', 'HH_DELTA_FLAG', 'KOMBIALTER']
    if information_level == 'Household':
        attributes_list += ['ALTER_KIND1', 'ALTER_KIND2', 'ALTER_KIND3', 'ALTER_KIND4',
                            'ANZ_KINDER', 'ANZ_STATISTISCHE_HAUSHALTE',
                            'D19_BANKEN_ANZ_12', 'D19_BANKEN_ANZ_24', 'D19_KONSUMTYP_MAX',
                            'D19_TELKO_ANZ_12', 'D19_TELKO_ANZ_24', 'D19_TELKO_ONLINE_QUOTE_12', 'D19_VERSAND_ANZ_12',
                            'D19_VERSAND_ANZ_24', 'D19_VERSI_ANZ_12', 'D19_VERSI_ANZ_24', 'D19_VERSI_ONLINE_QUOTE_12',
                            'KK_KUNDENTYP', 'KONSUMZELLE', 'UMFELD_ALT', 'STRUKTURTYP',
                            'UMFELD_JUNG', 'UNGLEICHENN_FLAG', 'VERDICHTUNGSRAUM',
                            'VHA', 'VHN', 'VK_DHT4A', 'VK_DISTANZ', 'VK_ZG11']
    if information_level == 'PLZ8':
        attributes_list += ['KBA13_ANTG1', 'KBA13_ANTG2', 'KBA13_ANTG3', 'KBA13_ANTG4', 'KBA13_BAUMAX', 'KBA13_CCM_1401_2500',
                            'KBA13_CCM_3000', 'KBA13_CCM_3001', 'KBA13_GBZ', 'KBA13_HHZ', 'KBA13_KMH_210']
    if information_level == 'Person':
        attributes_list += ['ALTERSKATEGORIE_FEIN', 'AKT_DAT_KL',
                            'CJT_KATALOGNUTZER', 'CJT_TYP_1', 'CJT_TYP_2', 'CJT_TYP_3', 'CJT_TYP_4', 'CJT_TYP_5', 'CJT_TYP_6',
                            'DSL_FLAG', 'EINGEFUEGT_AM', 'EINGEZOGENAM_HH_JAHR',
                            'RT_KEIN_ANREIZ', 'RT_SCHNAEPPCHEN', 'RT_UEBERGROESSE', 'SOHO_KZ']
    if information_level == 'Microcell (RR4_ID)':
        attributes_list += ['CAMEO_INTL_2015']
    if information_level == 'Postcode':
        attributes_list += ['GEMEINDETYP', 'EXTSEL992']
    if information_level == 'RR1_ID':
        attributes_list += ['MOBI_RASTER']

    return attributes_list

def plot_heatmap_isna(information_columns, title, data):
    plt.figure(figsize=(10,5))
    sns.heatmap(data[information_columns].isna(), cbar=False)
    plt.title(title)

    plt.show()


from pathlib import Path
from itertools import zip_longest

f = open(Path(__file__).resolve().parent / 'input.txt')

# eval would have been easier ...
def read_packet(s):
  def read(s,p):
    if s[p] == '[':
      return read_list(s,p)
    return read_number(s,p)
  def read_number(s,p):
    e = p
    while s[e].isdigit():
      e += 1
    return int(s[p:e]),e
  def read_list(s,p):
    l = []
    p += 1
    while True:
      if s[p] == ',':
        p += 1
        continue
      if s[p] == ']':
        p += 1
        break
      i,p = read(s,p)
      l.append(i)
    return l,p
  return read_list(s,0)[0]

input = []
i = iter(f)
while True:
  n = next(i)
  a = read_packet(n.strip())
  n = next(i)
  b = read_packet(n.strip())
  input.append((a,b))
  try:
    next(i)
  except:
    break

def cmp(a,b):
  if isinstance(a,int) and isinstance(b,int):
    if a < b:
      return True
    if a > b:
      return False
    return None
  if isinstance(a,list) and isinstance(b,list):
    for aa,bb in zip_longest(a,b):
      if aa is None:
        return True
      if bb is None:
        return False
      c = cmp(aa,bb)
      if c is not None:
        return c
    return None
  if isinstance(a,int):
    return cmp([a],b)
  return cmp(a,[b])

sum = 0
for i,(a,b) in enumerate(input):
  if cmp(a,b):
    sum += i+1
print(sum)

from functools import cmp_to_key

div1,div2 = [[2]],[[6]]
input = [a for a,b in input] + [b for a,b in input] + [div1,div2]

def cmp2(a,b):
  c = cmp(a,b)
  if c is None:
    raise RuntimeError('dafuq')
  return -1 if c else 1

s = sorted(input, key=cmp_to_key(cmp2))
print((s.index(div1)+1) * (s.index(div2)+1))


from django import views
from django.contrib.auth.mixins import LoginRequiredMixin
from django.shortcuts import render

from .forms import ProfileChangeForm

class ProfileView(LoginRequiredMixin, views.View):
    template_name = "LK.html"
    form_class = ProfileChangeForm

    def get(self, request, *args, **kwargs):
        user = request.user
        user_tours = user.reserved_tours.all()
        user_reviews = user.review_set.all()
        data = {
            "user_tours": user_tours,
            "user_reviews": user_reviews
        }
        return render(request, self.template_name, context=data)

    def post(self, request, *args, **kwargs):
        form = self.form_class(request.POST)
        if form.is_valid():
            user = request.user
            user.last_name = form.cleaned_data['last_name']
            user.first_name = form.cleaned_data['first_name']
            user.patronymic_name = form.cleaned_data['patronymic_name']
            user.phone = form.cleaned_data['phone']
            user.save()

            return render(request, self.template_name)
        else:
            return render(request, self.template_name, {'form': form})


from bs4 import BeautifulSoup
import sqlite3
import urllib.request
import urllib.error
import re
import xlwt
def main():
    baseurl = 'https://www.liepin.com/zhaopin/?sfrom=click-pc_homepage-centre_searchbox-search_new&dqs=010&key=JAVA'
    datalist = getdata(baseurl)
    savepath = ".\\职位信息"
    print(datalist)
    # 伪装请求
def askurl(url):
    # 请求头
    head = {
        'accept': '*/*',
        'accept - encoding': 'gzip, deflate, br',
        'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
        'x-requested-with': 'XMLHttpRequest',
        'referer':'https://www.liepin.com/zhaopin/?sfrom=click-pc_homepage-centre_searchbox-search_new&dqs=010&key=JAVA',
        'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36 Edg/88.0.705.50',

    }
    #构造request
    request = urllib.request.Request(url=url, headers=head)
    html = ""
    try:
    #构造response
        response = urllib.request.urlopen(request)
        html = response.read().decode("utf-8")
    except urllib.error.URLError as e:
        if hasattr(e, "code"):
            print(e.code)
        if hasattr(e, "reason"):
            print(e.reason)
def getdata(baseurl):
    data = [ ]
    for i in range(0,1):
        url = baseurl + "?page=2&ka=page-"+str(i)
        html = askurl(url)
        soup = BeautifulSoup(html, "html.parser")
        print(html)
        # 获取职位列表
        for item in soup.find_all('div', class_="job-content"):
            # 保存职位信息
            data = []
            item = str(item)
            # 添加公司名字，月薪，技能要求，工作地点，职位描述到列表
            companyname = re.findall(findcompanyname, item)[0]
            data.append(companyname)
            salary = re.findall(findsalary, item)[0]
            data.append(salary)
            workposition = re.findall(findworkposition, item)[0]
            data.append(workposition)
    return data
# 保存数据到excel
def savedata(datalist,savepath):
    print("...")
    book = xlwt.Workbook(encodeing="utf-8",style_copression=0)
    sheet = book.add_sheet('职位表单',cell_overwrite_ok=True)
    #定义列
    col = ("公司名字")
    for i in range(0,5):
        sheet.write(0,i,col[i])
    for i in range(0,):
        dta = datalist[i]
        for j in range(0,5):
            sheet.write(i+1,j,data[j])
    book.save(savepath)
if __name__ == "__main__":
    main()
    print("....")
    # init_db("movetest.db")
# 公司名字正则
findcompanyname = re.compile(r'
(.*?)')
# 月薪正则
findsalary = re.compile(r'(.*?)')
# 工作地点正则
findworkposition = re.compile(r'(.*?)')
# 描述正则
# finddescription = re.compile()
# # def init_db(dbpath):
#     sql = '''
#         create table jobtq(
#         id integer primary key autoincrement,
#         findcompanyname text,
#         findsalary text,
#         findtechnologyrequest text,
#         findworkposition text,
#         finddescription text,
#         )
#     '''
#     conn = sqlite3.connect(dbpath)
#     cursor = conn.cursor()
#     cursor.execute(sql)
#     conn.commit()
#     conn.close()


import numpy as np
import pandas as pd

from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.pipeline import make_pipeline
from sklearn.linear_model import LinearRegression

from plots import plot_series
from sunspots import error, get_data, sklearn_formatting

class GaussianFeatures(BaseEstimator, TransformerMixin):
    def __init__(self, N, width_f=2.0):
        self.N = N
        self.width_f = width_f

    @staticmethod
    def _gauss_basis(x, y, width, axis=None):
        arg = (x - y) / width
        return np.exp(-0.5 * np.sum(arg ** 2, axis))

    def fit(self, X, y=None):
        # N centres along range
        self._centres = np.linspace(X.min(), X.max(), self.N)
        self._width = self.width_f * (self._centres[1] - self._centres[0])
        return self

    def transform(self, X):
        return self._gauss_basis(X[:, :, np.newaxis], self._centres, self._width, axis=1)

def linear(train, test, t=132):
    X_train, X_test = sklearn_formatting(train, test)

    gauss_model = make_pipeline(
        GaussianFeatures(40),
        LinearRegression(),
        )

    gauss_model.fit(X_train, train.values)
    y_fit = gauss_model.predict(X_train)

    # predict a cycle
    y_pred = gauss_model.predict(X_test)
    rmse = error(test.values, y_pred)
    return y_fit, y_pred, rmse

if __name__ == "__main__":

    df_train, df_test = get_data()
    lin_y, lin_y_pred, lin_rmse = linear(df_train, df_test)

    plot_series(df_train, df_test, lin_y, lin_y_pred)


import glob
import io
import logging
import re
import textwrap
import traceback
from contextlib import redirect_stdout

import discord
from discord.ext import commands
from discord.ext.commands import Cog, Bot, Context

from cogs.commands import settings
from utils import embeds
from utils.record import record_usage

# Enabling logs
log = logging.getLogger(__name__)

class AdministrationCog(Cog):
    """ Administration Cog Cog """

    def __init__(self, bot):
        self.bot = bot
        self._last_result = None

    def _cleanup_code(self, content):
        """Automatically removes code blocks from the code."""
        # remove ```py\n```
        if content.startswith('```') and content.endswith('```'):
            return '\n'.join(content.split('\n')[1:-1])

        # remove `foo`
        return content.strip('` \n')

    @commands.before_invoke(record_usage)
    @commands.group(aliases=["u", "ul"])
    async def utilities(self, ctx):
        if ctx.invoked_subcommand is None:
            # Send the help command for this group
            await ctx.send_help(ctx.command)

    @commands.is_owner()
    @utilities.command(name="ping")
    async def ping(self, ctx):
        """Returns the Discord WebSocket latency."""
        await ctx.send(f"{round(self.bot.latency * 1000)}ms.")

    @commands.is_owner()
    @utilities.command(name="say")
    async def say(self, ctx, *, args):
        """Echos the input argument."""
        await ctx.send(args)

    @commands.is_owner()
    @utilities.command(name="eval")
    async def eval(self, ctx, *, body: str):
        """Evaluates input as Python code."""
        # Required environment variables.
        env = {
            'bot': self.bot,
            'ctx': ctx,
            'channel': ctx.channel,
            'author': ctx.author,
            'guild': ctx.guild,
            'message': ctx.message,
            'embeds': embeds,
            '_': self._last_result
        }
        # Creating embed.
        embed = discord.Embed(title="Evaluating.", color=0xb134eb)
        env.update(globals())

        # Calling cleanup command to remove the markdown traces.
        body = self._cleanup_code(body)
        embed.add_field(
            name="Input:", value=f"```py\n{body}\n```", inline=False)
        # Output stream.
        stdout = io.StringIO()

        # Exact code to be compiled.
        to_compile = f'async def func():\n{textwrap.indent(body, "  ")}'

        try:
            # Attempting execution
            exec(to_compile, env)
        except Exception as e:
            # In case there's an error, add it to the embed, send and stop.
            errors = f'```py\n{e.__class__.__name__}: {e}\n```'
            embed.add_field(name="Errors:", value=errors, inline=False)
            await ctx.send(embed=embed)
            return errors

        func = env['func']
        try:
            with redirect_stdout(stdout):
                ret = await func()
        except Exception:
            # In case there's an error, add it to the embed, send and stop.
            value = stdout.getvalue()
            errors = f'```py\n{value}{traceback.format_exc()}\n```'
            embed.add_field(name="Errors:", value=errors, inline=False)
            await ctx.send(embed=embed)

        else:
            value = stdout.getvalue()
            try:
                await ctx.message.add_reaction('\u2705')
            except:
                pass

            if ret is None:
                if value:
                    # Output.
                    output = f'```py\n{value}\n```'
                    embed.add_field(
                        name="Output:", value=output, inline=False)
                    await ctx.send(embed=embed)
            else:
                # Maybe the case where there's no output?
                self._last_result = ret
                output = f'```py\n{value}{ret}\n```'
                embed.add_field(name="Output:", value=output, inline=False)
                await ctx.send(embed=embed)

    @commands.is_owner()
    @utilities.command(name="reload")
    async def reload_cog(self, ctx: commands.Context, name_of_cog: str = None):
        """ Reloads specified cog or all cogs. """

        regex = r"(?<=<).*(?=\..* object at 0x.*>)"
        if name_of_cog is not None and name_of_cog in ctx.bot.cogs:
            # Reload cog if it exists.
            cog = re.search(regex, str(ctx.bot.cogs[name_of_cog]))
            try:
                self.bot.reload_extension(cog.group())

            except commands.ExtensionError as e:
                await ctx.message.add_reaction("❌")
                await ctx.send(f'{e.__class__.__name__}: {e}')

            else:
                await ctx.message.add_reaction("✔")
                await ctx.send(f"Reloaded `{cog.group()}` module!")

        elif name_of_cog is None:
            # Reload all the cogs in the folder named cogs.
            # Skips over any cogs that start with '__' or do not end with .py.
            cogs = []
            try:
                for cog in glob.iglob("cogs/**/[!^_]*.py", recursive=True):
                    if "\\" in cog:  # Pathing on Windows.
                        self.bot.reload_extension(cog.replace("\\", ".")[:-3])
                    else:  # Pathing on Linux.
                        self.bot.reload_extension(cog.replace("/", ".")[:-3])
            except commands.ExtensionError as e:
                await ctx.message.add_reaction("❌")
                await ctx.send(f'{e.__class__.__name__}: {e}')

            else:
                await ctx.message.add_reaction("✔")
                await ctx.send("Reloaded all modules!")
        else:
            await ctx.message.add_reaction("❌")
            await ctx.send("Module not found, check spelling, it's case sensitive.")

    @commands.is_owner()
    @commands.bot_has_permissions(embed_links=True, send_messages=True)
    @commands.before_invoke(record_usage)
    @commands.command(name="rules")
    async def rules(self, ctx: Context):
        """ Generates the #rules channel embeds. """

        # Captain Karen header image embed
        embed = embeds.make_embed(color="quotes_grey")
        embed.set_image(url="https://i.imgur.com/Yk4kwZy.gif")
        await ctx.send(embed=embed)

        # The actual rules embed
        embed = embeds.make_embed(title="📃  Discord Server Rules", color="quotes_grey", description="This list is not all-encompassing and you may be actioned for a reason outside of these rules. Use common sense when interacting in our community.")
        embed.add_field(name="Rule 1: Do not send copyright-infringing material.", inline=False, value="> Linking to torrents, pirated stream links, direct download links, or uploading files over Discord puts our community at risk of being shut down. We are a discussion community, not a file-sharing hub.")
        embed.add_field(name="Rule 2: Be courteous and mindful of others.", inline=False, value="> Do not engage in toxic behavior such as spamming, derailing conversations, attacking other users, or attempting to instigate drama. Bigotry will not be tolerated. Avoid problematic avatars, usernames, or nicknames.")
        embed.add_field(name="Rule 3: Do not post self-promotional content.", inline=False, value="> We are not a billboard nor the place to advertise your Discord server, app, website, service, etc.")
        embed.add_field(name="Rule 4: Do not post unmarked spoilers.", inline=False, value="> Use spoiler tags and include what series or episode your spoiler is in reference to outside the spoiler tag so people don't blindly click a spoiler.")
        embed.add_field(name="Rule 5: Do not backseat moderate.", inline=False, value="> If you see someone breaking the rules or have something to report, please submit a <#829861810999132160> ticket.")
        embed.add_field(name="Rule 6: Do not abuse pings.", inline=False, value="> Do not ping staff outside of conversation unless necessary. Do not ping VIP users for questions or help with their service. Do not spam or ghost ping other users.")
        embed.add_field(name="Rule 7: Do not beg, buy, sell, or trade.", inline=False, value="> This includes, but is not limited to, server ranks, roles, permissions, giveaways, private community invites, or any digital or physical goods.")
        embed.add_field(name="Rule 8: Follow the Discord Community Guidelines and Terms of Service.", inline=False, value="> The Discord Community Guidelines and Terms of Service govern all servers on the platform. Please familarize yourself with them and the restrictions that come with them. \n> \n> https://discord.com/guidelines \n> https://discord.com/terms")
        await ctx.send(embed=embed)

        # /r/animepiracy links embed
        embed = embeds.make_embed(title="🔗  Our Links", color="quotes_grey")
        embed.add_field(name="Reddit:", inline=True, value="> [/r/animepiracy](https://reddit.com/r/animepiracy)")
        embed.add_field(name="Discord:", inline=True, value="> [discord.gg/piracy](https://discord.gg/piracy)")
        embed.add_field(name="Index:", inline=True, value="> [piracy.moe](https://piracy.moe)")
        embed.add_field(name="Wiki:", inline=True, value="> [wiki.piracy.moe](https://wiki.piracy.moe)")
        embed.add_field(name="Seadex:", inline=True, value="> [releases.moe](https://releases.moe)")
        embed.add_field(name="GitHub:", inline=True, value="> [github.com/ranimepiracy](https://github.com/ranimepiracy)")
        embed.add_field(name="Twitter:", inline=True, value="> [@ranimepiracy](https://twitter.com/ranimepiracy)")
        embed.add_field(name="Uptime Status:", inline=True, value="> [status.piracy.moe](https://status.piracy.moe/)")
        await ctx.send(embed=embed)

        # Clean up the command invoker
        await ctx.message.delete()

    @commands.is_owner()
    @commands.bot_has_permissions(embed_links=True, send_messages=True)
    @commands.before_invoke(record_usage)
    @commands.command(name="createticketembed")
    async def create_ticket_embed(self, ctx: Context):
        embed = embeds.make_embed(title="🎫 Create a new modmail ticket",
                                  description="Click the react below to create a new modmail ticket.",
                                  color="default")
        embed.add_field(name="Warning:", value="Serious inquiries only. Abuse may result in warning or ban.")
        spawned = await ctx.send(embed=embed)
        await spawned.add_reaction("🎫")
        await ctx.message.delete()

    @commands.is_owner()
    @commands.bot_has_permissions(embed_links=True, send_messages=True)
    @commands.before_invoke(record_usage)
    @commands.command(name="createcolorrolesembed", aliases=['ccre'])
    async def create_color_roles_embed(self, ctx: Context):
        embed = discord.Embed(description=f"You can react to one of the squares below to be assigned a colored user role. If you are interested in a different color, you can become a <@&{settings.get_value('role_server_booster')}> to receive a custom colored role.")
        msg = await ctx.send(embed=embed)

        # API call to fetch all the emojis to cache, so that they work in future calls
        emotes_guild = await ctx.bot.fetch_guild(settings.get_value("emoji_guild_id"))
        emojis = await emotes_guild.fetch_emojis()

        await msg.add_reaction(":redsquare:805032092907601952")
        await msg.add_reaction(":orangesquare:805032107952308235")
        await msg.add_reaction(":yellowsquare:805032120971165709")
        await msg.add_reaction(":greensquare:805032132325801994")
        await msg.add_reaction(":bluesquare:805032145030348840")
        await msg.add_reaction(":pinksquare:805032162197635114")
        await msg.add_reaction(":purplesquare:805032172074696744")
        await ctx.message.delete()

    @commands.is_owner()
    @commands.bot_has_permissions(embed_links=True, send_messages=True)
    @commands.before_invoke(record_usage)
    @commands.command(name="createassignablerolesembed", aliases=['care'])
    async def create_assignable_roles_embed(self, ctx: Context):
        role_assignment_text = """
        You can react to one of the emotes below to assign yourself an event role.

        🎁  <@&832528733763928094> - Receive giveaway pings.
        📢  <@&827611682917711952> - Receive server announcement pings.
        📽  <@&831999443220955136> - Receive group watch event pings.
        <:kakeraW:830594599001129000>  <@&832512304334766110> - Receive Mudae event and season pings.
        🧩  <@&832512320306675722> - Receive Rin event pings.
        <:pickaxe:831765423455993888>  <@&832512327731118102> - Receive Minecraft server related pings.
        """
        embed = discord.Embed(description=role_assignment_text)
        msg = await ctx.send(embed=embed)

        # API call to fetch all the emojis to cache, so that they work in future calls
        emotes_guild = await ctx.bot.fetch_guild(settings.get_value("emoji_guild_id"))
        await emotes_guild.fetch_emojis()

        await msg.add_reaction("🎁")
        await msg.add_reaction("📢")
        await msg.add_reaction("📽")
        await msg.add_reaction(":kakeraW:830594599001129000")
        await msg.add_reaction("🧩")
        await msg.add_reaction(":pickaxe:831765423455993888")
        await ctx.message.delete()

def setup(bot: Bot) -> None:
    """ Load the AdministrationCog cog. """
    bot.add_cog(AdministrationCog(bot))
    log.info("Commands loaded: administration")


#==== MOVIE ADMIN VIEWS ====

from django.shortcuts import render, get_object_or_404

from django.urls import reverse
from django.http import HttpResponseRedirect
from django.contrib.auth import authenticate

from movies.models import *

from Metflix.settings import MEDIA_ROOT, MEDIA_URL
import os

# Create your views here.

def index(request):
    print (MEDIA_ROOT[0])
    print (MEDIA_ROOT[1])
    movies = Movie.objects.all()
    genres = Genre.objects.all()

    context = {
        'test': 'test'
    }

    return render(request, 'movie_admin/index.html', context)

def add_movies(request):
    MOVIE_ROOT = os.path.join(os.path.join(MEDIA_ROOT[0], MEDIA_ROOT[1]), 'movies')

    # Check movie root dir for new movie folders
    # and adds them to database
    movie_list_dirs = os.listdir(MOVIE_ROOT)
    for movie in movie_list_dirs:
        # Check if movie alredy is in database
        # if so do not add movie
        try:
            movie_db = Movie.objects.get(title = movie)
        except:
            m = Movie.objects.create(title = movie, type = 1)
            m.save()
        else:
            print ('Movie excists')

    # Check al database entrys to the movie root dir
    # if there is not dir with movie name
    # Delete from database
    movie_list_db = Movie.objects.all()
    for movie in movie_list_db:
        if not os.path.isdir(os.path.join(MOVIE_ROOT, movie.title)):
            movie.delete()

    return HttpResponseRedirect(reverse('movie_admin:index'))

def add_series(request):

    SERIES_ROOT = os.path.join(MEDIA_ROOT, 'series')

    series_list_dir = os.listdir(SERIES_ROOT)

    for series in series_list_dir:

        try:
            series_db = Movie.objects.get(title = series)
        except:
            s = Movie.objects.create(title = movie, type = 2)
            s.save()
        else:
            print("Series Excists")

        # Search the series dir for season dirs
        season_list_dir = os.listdir(os.path.join(SERIES_ROOT, series))

        for season in season_list_dir:

            episode_db = Episode.objects.all()
            season_db = episode_db.filter()

    return HttpResponseRedirect(reverse('movie_admin:index'))


# -*- coding: utf-8 -*-

from builtins import object
from PyQt5 import QtCore, QtGui, QtWidgets

try:
    _fromUtf8 = QtCore.QString.fromUtf8
except AttributeError:
    _fromUtf8 = lambda s: s

class Ui_qgsnewhttpconnectionbase(object):
    def setupUi(self, qgsnewhttpconnectionbase):
        qgsnewhttpconnectionbase.setObjectName(_fromUtf8("qgsnewhttpconnectionbase"))
        qgsnewhttpconnectionbase.resize(642, 153)
        self.buttonBox = QtWidgets.QDialogButtonBox(qgsnewhttpconnectionbase)
        self.buttonBox.setGeometry(QtCore.QRect(280, 110, 341, 32))
        self.buttonBox.setFocusPolicy(QtCore.Qt.NoFocus)
        self.buttonBox.setOrientation(QtCore.Qt.Horizontal)
        self.buttonBox.setStandardButtons(QtWidgets.QDialogButtonBox.Cancel|QtWidgets.QDialogButtonBox.Ok)
        self.buttonBox.setObjectName(_fromUtf8("buttonBox"))
        self.label_NewSrvName = QtWidgets.QLabel(qgsnewhttpconnectionbase)
        self.label_NewSrvName.setGeometry(QtCore.QRect(20, 33, 91, 17))
        self.label_NewSrvName.setObjectName(_fromUtf8("label_NewSrvName"))
        self.label_NewSrvUrl = QtWidgets.QLabel(qgsnewhttpconnectionbase)
        self.label_NewSrvUrl.setGeometry(QtCore.QRect(20, 75, 91, 17))
        self.label_NewSrvUrl.setObjectName(_fromUtf8("label_NewSrvUrl"))
        self.txt_NewSrvName = QtWidgets.QLineEdit(qgsnewhttpconnectionbase)
        self.txt_NewSrvName.setEnabled(True)
        self.txt_NewSrvName.setGeometry(QtCore.QRect(120, 30, 501, 27))
        self.txt_NewSrvName.setCursor(QtGui.QCursor(QtCore.Qt.IBeamCursor))
        self.txt_NewSrvName.setFocusPolicy(QtCore.Qt.ClickFocus)
        self.txt_NewSrvName.setObjectName(_fromUtf8("txt_NewSrvName"))
        self.txt_NewSrvUrl = QtWidgets.QLineEdit(qgsnewhttpconnectionbase)
        self.txt_NewSrvUrl.setGeometry(QtCore.QRect(120, 70, 501, 27))
        self.txt_NewSrvUrl.setObjectName(_fromUtf8("txt_NewSrvUrl"))

        self.retranslateUi(qgsnewhttpconnectionbase)
        self.buttonBox.accepted.connect(qgsnewhttpconnectionbase.accept)
        self.buttonBox.rejected.connect(qgsnewhttpconnectionbase.reject)
        QtCore.QMetaObject.connectSlotsByName(qgsnewhttpconnectionbase)

    def retranslateUi(self, qgsnewhttpconnectionbase):
        qgsnewhttpconnectionbase.setWindowTitle(QtWidgets.QApplication.translate("qgsnewhttpconnectionbase", "New WCPS Server ", None))
        self.label_NewSrvName.setText(QtWidgets.QApplication.translate("qgsnewhttpconnectionbase", "Server Name", None))
        self.label_NewSrvUrl.setText(QtWidgets.QApplication.translate("qgsnewhttpconnectionbase", "Server URL", None))


"""Point cloud generation functions to test benchmarks of rotation invariant functions
"""
from dataclasses import dataclass
from functools import lru_cache

import numpy as np
import torch
from numpy.linalg import norm
from scipy.spatial.transform import Rotation
from scipy.spatial.transform import Rotation as R

def to_torch_tensor(np_array):
    """
    convert a numpy array to a torch tensor by putting it to the appropriate type
    and dimension
    """
    return torch.tensor(np_array).unsqueeze(0).to(torch.float32)

def to_numpy_array(torch_tensor):
    """
    convert a torch tensor to a numpy array by putting it to the appropriate type
    and dimension
    """
    return torch_tensor.squeeze().detach().numpy()

# Rotation and Permutation matrices
def generate_permutation_matrix(N):
    """
    Generate a random permutation matrix
    """
    return np.random.permutation(np.identity(N))

def generate_rotation_matrix(theta=None, axis=None):
    """
    Return a rotation matrix corresponding to a rotation of amplitude theta
    around the provided axis. If an argument is set to None, a random value is assigned
    """
    if theta is None:
        theta = np.random.rand(1)[0] * 2 * np.pi
    if axis is None:
        axis = np.random.rand(3) - 0.5
        axis /= norm(axis)
    return Rotation.from_rotvec(theta * axis).as_matrix()

def rotate(*args):
    """
    Return the rotated version of each tensor passed in parameter using the same rotation
    matrix

    """
    Q = generate_rotation_matrix()
    return [points_tens @ Q for points_tens in args]

# Centering
def center_batch(batch):
    """center a batch

    Args:
        batch (torch.tensor): [n_batch x N_points x 1 x 3]

    Returns:
        [type]: [description]
    """
    mean_input = batch.mean(axis=1).unsqueeze(1)
    return batch - mean_input

def center(sample):
    """
    Center batch to have a barycenter around 0

    Args:
        sample (np.array): N x 3 sample
    """
    return sample - sample.mean(axis=0)

def get_points_on_sphere(N):
    """
    Generate N evenly distributed points on the unit sphere centered at
    the origin. Uses the 'Golden Spiral'.
    Code by Chris Colbert from the numpy-discussion list.
    """
    phi = (1 + np.sqrt(5)) / 2  # the golden ratio
    long_incr = 2 * np.pi / phi  # how much to increment the longitude

    dz = 2.0 / float(N)  # a unit sphere has diameter 2
    bands = np.arange(N)  # each band will have one point placed on it
    z = bands * dz - 1 + (dz / 2)  # the height z of each band/point
    r = np.sqrt(1 - z * z)  # project onto xy-plane
    az = bands * long_incr  # azimuthal angle of point modulo 2 pi
    x = r * np.cos(az)
    y = r * np.sin(az)
    return np.array((x, y, z)).transpose()

def get_angle(u, v):
    """
    Return the angle between 3d vectors u and v
    """
    cos_angle = np.dot(u, v) / norm(u) / norm(v)
    return np.arccos(np.clip(cos_angle, -1, 1))

def get_n_regular_rotations(N):
    """
    Return N regular rotations matrices

    Args:
        N (int): number of rotation matrixes to return
    """
    # generate random points on the unit sphere
    reg_points = get_points_on_sphere(N)

    # take the first vector as the reference one
    ref_vec = reg_points[0]

    # computation of angle and cross product as described here
    # https://math.stackexchange.com/questions/2754627/rotation-matrices-between-two-points-on-a-sphere
    thetas = [get_angle(ref_vec, v) for v in reg_points]
    cross_vecs = np.cross(ref_vec, reg_points)
    rotations = [
        (R.from_rotvec(theta * cross_vec).as_matrix())
        for theta, cross_vec in zip(thetas, cross_vecs)
    ]
    return rotations

# Gaussian Point cloud

def get_gaussian_point_cloud(N_pts):
    return np.random.rand(N_pts, 3)

# Spiral Point cloud

def get_asym_spiral(spiral_amp=1.0, width_factor=1.0, N_pts=40):
    """
    Generate a spiral with the given amplitude

    Args:
        spiral_amp (float, optional): spiral amplitude

    Returns:
        np.array: spiral point cloud
    """
    zline = np.linspace(0, spiral_amp, N_pts)
    xline = [(i + 4) * np.sin(i) / 10 *
             width_factor for i in zline * 4 * np.pi]
    yline = [(i + 4) * np.cos(i) / 10 *
             width_factor for i in zline * 4 * np.pi]
    return np.array([xline, yline, zline]).transpose()

def get_spiral(spiral_amp=1.0, N_pts=40):
    """
    Generate a spiral with the given amplitude

    Args:
        spiral_amp (float, optional): spiral amplitude

    Returns:
        np.array: spiral point cloud
    """
    zline = np.linspace(0, spiral_amp, N_pts)
    xline = np.sin(zline * 4 * np.pi)
    yline = np.cos(zline * 4 * np.pi)
    return np.array([xline, yline, zline]).transpose()

@dataclass(frozen=True)
class SpiralGenerator:
    spiral_amp: float = 1.0
    scaling: float = 1.0
    shift: float = 0.0
    asym: bool = False
    width_factor: float = 1.0
    centering: bool = False

    def __post_init__(self):
        assert(not (self.centering and self.shift != 0))

    @lru_cache(None)
    def generate(self,):
        if self.asym:
            points = get_asym_spiral(
                spiral_amp=self.spiral_amp, width_factor=self.width_factor)
        else:
            points = get_spiral(spiral_amp=self.spiral_amp)

        [xline, yline, zline] = points.transpose()
        zline *= self.scaling
        zline += self.shift
        points = np.array([xline, yline, zline]).transpose()
        if self.centering:
            points = center(points)
        return points

def get_custom_spiral(spiral_amp=1.0, scaling=1.0, shift=0.0, asym=False, centering=False):
    assert(not (centering and shift != 0))

    if asym:
        points = get_asym_spiral(spiral_amp=spiral_amp)
    else:
        points = get_spiral(spiral_amp=spiral_amp)

    [xline, yline, zline] = points.transpose()
    zline *= scaling
    zline += shift
    points = np.array([xline, yline, zline]).transpose()
    if centering:
        points = center(points)
    return points

def get_src_shifted_spirals(
    spiral_amp=1.0, shift=0.5, asym=False, center_input=False, center_target=False
):
    """
    Return vertical src spiral cloud point and its vertically shifted version.
    """
    # torch.set_default_dtype(torch.float64) # works best in float64
    if asym:
        points = get_asym_spiral(spiral_amp=spiral_amp)
    else:
        points = get_spiral(spiral_amp=spiral_amp)
    target_points = np.array([xline, yline, (zline + shift)]).transpose()
    if center_input:
        points = center(points)
    if center_target:
        target_points = center(target_points)
    return points, target_points

def get_src_scaled_spirals(
    spiral_amp=1.0, z_scale=3, asym=False, center_input=False, center_target=False
):
    if asym:
        points = get_asym_spiral(spiral_amp=spiral_amp)
    else:
        points = get_spiral(spiral_amp=spiral_amp)
    [xline, yline, zline] = points.transpose()
    target_points = np.array([xline, yline, zline * z_scale]).transpose()
    if center_input:
        points = center(points)
    if center_target:
        target_points = center(target_points)
    return points, target_points

def get_src_inverted_spirals(spiral_amp=1.0):
    """
    Return inverted spiral with same positions
    """
    points = get_spiral(spiral_amp=spiral_amp)
    [xline, yline, zline] = points
    target_points = np.array([-xline, yline, zline]).transpose()

    return points, target_points


from django.urls import path
from .views import (
    PdfextListView,
    PdfextDetailView,
    PdfextCreateView,
    PdfextUpdateView,
    PdfextDeleteView,
    UserPdfextListView,
    PdfextConvertView,
)
from . import views

urlpatterns = [
    path('', PdfextListView.as_view(), name='pdfext-home'),
    path('user/<str:username>', UserPdfextListView.as_view(), name='user-pdfexts'),
    path('pdfext/<int:pk>/', PdfextDetailView.as_view(), name='pdfext-detail'),
    path('pdfext/new/', PdfextCreateView.as_view(), name='pdfext-create'),
    path('pdfext/<int:pk>/update/', PdfextUpdateView.as_view(), name='pdfext-update'),
    path('pdfext/<int:pk>/delete/', PdfextDeleteView.as_view(), name='pdfext-delete'),
    path('pdfext/<int:pk>/convert/', PdfextConvertView.as_view(), name='pdfext-convert'),
    path('media/Files/<int:pk>',PdfextDeleteView.as_view(),name='pdfext-delete' ),
    path('downloadjson/<int:pk>',views.downloadjson, name="downloadjson"),
    path('search/',views.search,name='search' ),
    path('about/', views.about, name='pdfext-about'),
]


from itertools import groupby
lis = [1, 2, 2, 3, 4, 4, 4, 4]
print(list(groupby(lis)))

things = [("animal", "bear"), ("animal", "duck"), ("plant", "cactus"), ("vehicle", "speed boat"), ("vehicle", "school bus")]

for key, group in groupby(things, lambda x: x[0]):
    for thing in group:
        print("A %s is a %s." % (thing[1], key))
    print(" ")

for key, group in groupby(lis, lambda x:x):
    count = 0
    for item in group:
        count += 1
    print("The key is "+ str(key) +" the count is "+str(count))


from selenium import webdriver
import time
import math

try:
    browser = webdriver.Chrome()
    link = "http://suninjuly.github.io/redirect_accept.html"
    browser.get(link)
    browser.find_element_by_tag_name("button").click()
    #confirm = browser.switch_to.alert
    #confirm.accept()

    windows = browser.window_handles
    current_window = browser.current_window_handle

    for win in windows:
        if current_window == win:
            print(win, " with current index: ", windows.index(win))
        else:
            print(win, " with index: ", windows.index(win))
    new_window = browser.window_handles[1]
    browser.switch_to.window(new_window)

    x = browser.find_element_by_id("input_value").text
    print(x)
    y = str(math.log(abs(12*math.sin(int(x)))))
    browser.find_element_by_id("answer").send_keys(y)
    browser.find_element_by_tag_name("button").click()

finally:
    time.sleep(7)
    browser.close()
    browser.quit()