# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.4.2
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # This notebook is used to model how the volume of data changes at different points in the pipeline
#
# ### The different datasets are as follows:
# - unioned_ehr (submitted directly from sites - no cleaning rules applied)
# - combined datasets
# - deidentified datasets
# #
# ### The dimensions of 'data volume' are as follows (for each table):
# - number of participants
# - number of records

from google.cloud import bigquery
# %reload_ext google.cloud.bigquery
client = bigquery.Client()
# %load_ext google.cloud.bigquery

# %matplotlib inline
import bq_utils
import utils.bq
from notebooks import parameters
import pandas as pd
import numpy as np
import six
import matplotlib.pyplot as plt
import operator

# +
unioned = parameters.UNIONED_Q4_2019
combined = parameters.COMBINED_Q4_2019
deid = parameters.DEID_Q4_2019

print(f"""
Unioned Dataset: {unioned}
Combined Dataset: {combined}
De-ID Dataset: {deid}
""")

# -

# ## Below are the functions that can be used to create graphs for visualization purposes

def create_dicts_w_info(df, x_label, column_label):
    """
    This function is used to create a dictionary that can be easily converted to a
    graphical representation based on the values for a particular dataframe

    Parameters
    ----------
    df (dataframe): dataframe that contains the information to be converted

    x_label (string): the column of the dataframe whose rows will then be converted
        to they keys of a dictionary

    column_label (string): the column that contains the data quality metric being
        investigated

    Returns
    -------
    data_qual_info (dictionary): has the following structure

        keys: the column for a particular dataframe that represents the elements that
            whose data quality is being compared (e.g. HPOs, different measurement/unit
            combinations)

        values: the data quality metric being compared
    """
    rows = df[x_label].unique().tolist()

    data_qual_info = {}

    for row in rows:
        sample_df = df.loc[df[x_label] == row]

        data = sample_df.iloc[0][column_label]

        data_qual_info[row] = data

    return data_qual_info

def create_graphs(info_dict, xlabel, ylabel, title, img_name, color,
                  total_diff_color, turnoff_x):
    """
    Function is used to create a bar graph for a particular dictionary with information about
    data quality

    Parameters
    ----------
    info_dict (dictionary): contains information about data quality. The keys for the dictionary
        will serve as the x-axis labels whereas the values should serve as the 'y-value' for the
        particular bar

    xlabel (str): label to display across the x-axis

    ylabel (str): label to display across the y-axis

    title (str): title for the graph

    img_name (str): image used to save the image to the local repository

    color (str): character used to specify the colours of the bars

    total_diff_color (bool): indicates whether or not the last bar should be coloured red (
        as opposed to the rest of the bars on the graph). This is typically used when the ultimate
        value of the dictionary is of particular important (e.g. representing an 'aggregate' metric
        across all of the sites)

    turnoff_x (bool): used to disable the x-axis labels (for each of the bars). This is typically used
        when there are so many x-axis labels that they overlap and obscure legibility
    """
    bar_list = plt.bar(range(len(info_dict)),
                       list(info_dict.values()),
                       align='center',
                       color=color)

    # used to change the color of the 'aggregate' column; usually implemented for an average
    if total_diff_color:
        bar_list[len(info_dict) - 1].set_color('r')

    if not turnoff_x:
        plt.xticks(range(len(info_dict)),
                   list(info_dict.keys()),
                   rotation='vertical')
    else:
        plt.xticks([])

    plt.ylabel(ylabel)
    plt.xlabel(xlabel)
    plt.title(title)
    #plt.show()
    plt.savefig(img_name, bbox_inches="tight")

def render_mpl_table(data,
                     col_width=15,
                     row_height=0.625,
                     font_size=12,
                     header_color='#40466e',
                     row_colors=['#f1f1f2', 'w'],
                     edge_color='w',
                     bbox=[0, 0, 1, 1],
                     header_columns=0,
                     ax=None,
                     **kwargs):
    """
    Function is used to improve the formatting / image quality of the output. The
    parameters can be changed as needed/desired.
    """

    # the np.array added to size is the main determinant for column dimensions
    if ax is None:
        size = (np.array(data.shape[::-1]) + np.array([2, 1])) * np.array(
            [col_width, row_height])
        fig, ax = plt.subplots(figsize=size)
        ax.axis('off')

    mpl_table = ax.table(cellText=data.values,
                         bbox=bbox,
                         colLabels=data.columns,
                         **kwargs)

    mpl_table.auto_set_font_size(False)
    mpl_table.set_fontsize(font_size)

    for k, cell in six.iteritems(mpl_table._cells):
        cell.set_edgecolor(edge_color)
        if k[0] == 0 or k[1] < header_columns:
            cell.set_text_props(weight='bold', color='w')
            cell.set_facecolor(header_color)
        else:
            cell.set_facecolor(row_colors[k[0] % len(row_colors)])
    return ax

def create_pie_chart(dataframe, title, img_name):
    """
    Function is used to create a pie chart that can show how much each site contributes
    to the overall 'drop' between the unioned and combined datasets

    Function also saves the outputted pie chart to the current directory

    Parameters
    ----------
    dataframe (df): dataframe for a particular table. shows the following for
                    HPOs that uploaded data:

        a. the number of rows in the unioned dataset
        b. the number of rows in the combined dataset
        c. the total 'drop' of rows across unioned to combined, expressed as a percentage
        d. the relative 'contribution' of each site to the overall drop from unioned to
           combined

    title (str): title of the graph

    img_name (str): title of the image to be saved
    """
    hpo_list = dataframe['source_hpo'].tolist()[1:]  # do not take 'total'
    percent_of_drop = dataframe['percent_of_drop'].tolist()[1:]

    labels = []

    # creating the labels for the graph
    for hpo, perc in zip(hpo_list, percent_of_drop):
        string = '{}, {}%'.format(hpo, perc)
        labels.append(string)

    wedges = [0.1] * len(labels)

    plt.pie(percent_of_drop,
            labels=None,
            shadow=True,
            startangle=140,
            explode=wedges)

    plt.axis('equal')
    plt.title(title)
    plt.legend(bbox_to_anchor=(0.5, 0.75, 1.0, 0.85), labels=labels)

    plt.savefig(img_name, bbox_inches="tight")

    plt.show()

# ## Below are functions that can be used to create 'aggregate' dataframes for different tables

def generate_query(dataset, person_var, record_var, table_name, field_name):
    """
    This function is used to:
        a. generate a string that can be fed into BigQuery
        b. create a dataframe that contains information about the number of people and
           records for a particular dataset

    Parameters
    ----------
    dataset (string): name of the dataset that will be queried (originally from the
                      parameters file)

    person_var (string): variable that dictates how the 'number of people' will be
                         displayed in the resultant dataframe

    record_var (string): variable that dictates how the 'number of records' will be
                         displayed in the resultant dataframe

    table_name (string): represents the table that is being queried

    field_name (string): represents the field that should count the number of records
                         for a particular dataset/table combination. this is usually
                         'table name'_id

    Returns
    -------
    dataframe (df): dataframe with the information specified at the beginning of the
                    docstring
    """
    query = """
        SELECT
        DISTINCT
        COUNT(DISTINCT p.person_id) as {person_var}, COUNT(DISTINCT x.{field_name}) as {record_var}
        FROM
        `{dataset}.{table_name}` x
        JOIN
        `{dataset}.person` p
        ON
        x.person_id = p.person_id
        ORDER BY {person_var} DESC
    """.format(person_var=person_var,
               field_name=field_name,
               table_name=table_name,
               record_var=record_var,
               dataset=dataset)

    dataframe = pd.io.gbq.read_gbq(query, dialect='standard')

    return (dataframe)

def extract_first_int_from_series(series):
    """
    Function is used to extract the first integer from a Pandas series object.

    Parameters
    ----------
    series (series): Pandas series object

    Returns
    -------
    integer (int): the first integer from a Pandas series object
    """

    series_as_list = series.tolist()

    first_int = series_as_list[0]

    return first_int

def create_aggregate_table_df(unioned, combined, deid, unioned_persons_string,
                              combined_persons_string, deid_persons_string,
                              unioned_records_string, combined_records_string,
                              deid_records_string, person_string,
                              record_string):
    """
    Function is used to create a dataframe that can display the 'drop off' of records across multiple
    stages of the pipeline.

    Parameters:
    -----------

    unioned (dataframe): contains information regarding the number of persons and record in the unioned
        dataset

    combined (dataframe): contains information regarding the number of persons and record in the combined
        dataset

    deid (dataframe): contains information regarding the number of persons and record in the deid
        dataset

    unioned_person_string (str): column name to determine the number of persons in the unioned dataset

    combined_person_string (str): column name to determine the number of persons in the combined dataset

    deid_person_string (str): column name to determine the number of persons in the deid dataset

    unioned_records_string (str): column name to determine the number of records in the unioned dataset

    combined_records_string (str): column name to determine the number of records in the combined dataset

    deid_records_string (str): column name to determine the number of records in the deid dataset

    person_string (str): row title to indicate the person drop for each stage of the pipeline

    record_string (str): row title to indicate the record drop for each stage of the pipeline

    Returns:
    --------
    df (dataframe): contains information about the record and person count drop across each stage of
                    the pipeline
    """

    unioned_num_persons = extract_first_int_from_series(
        unioned[unioned_persons_string])
    combined_num_persons = extract_first_int_from_series(
        combined[combined_persons_string])
    deid_num_persons = extract_first_int_from_series(deid[deid_persons_string])

    unioned_num_records = extract_first_int_from_series(
        unioned[unioned_records_string])
    combined_num_records = extract_first_int_from_series(
        combined[combined_records_string])
    deid_num_records = extract_first_int_from_series(deid[deid_records_string])

    unioned_combined_person_drop = unioned_num_persons - combined_num_persons
    combined_deid_person_drop = combined_num_persons - deid_num_persons
    total_person_drop = unioned_combined_person_drop + combined_deid_person_drop
    total_person_drop_percent = round(
        total_person_drop / unioned_num_persons * 100, 2)

    unioned_combined_record_drop = unioned_num_records - combined_num_records
    combined_deid_record_drop = combined_num_records - deid_num_records
    total_record_drop = unioned_combined_record_drop + combined_deid_record_drop
    total_record_drop_percent = round(
        total_record_drop / unioned_num_records * 100, 2)

    data = [[
        unioned_combined_person_drop, combined_deid_person_drop,
        total_person_drop, total_person_drop_percent
    ],
            [
                unioned_combined_record_drop, combined_deid_record_drop,
                total_record_drop, total_record_drop_percent
            ]]

    df = pd.DataFrame(data,
                      columns=[
                          'Unioned/Combined Drop', 'Combined/De-ID Drop',
                          'Total Drop', 'Total Drop (%)'
                      ],
                      index=[person_string, record_string])

    return df

# ## Let's keep a dictionary so we can log the percent drop across all of the tables

# +
table_order = ['Condition', 'Visit', 'Procedure', 'Observation', 'Drug']

percent_drops = {'person_drop': [], 'record_drop': []}
# -

# ## Looking at how volume varies in the condition_occurrence table

condition_table_name = 'condition_occurrence'
condition_field_name = 'condition_occurrence_id'

condition_unioned = generate_query(dataset=unioned,
                                   person_var='unioned_condition_num_persons',
                                   record_var='unioned_condition_num_records',
                                   table_name=condition_table_name,
                                   field_name=condition_field_name)

condition_unioned

condition_combined = generate_query(dataset=combined,
                                    person_var='combined_condition_num_persons',
                                    record_var='combined_condition_num_records',
                                    table_name=condition_table_name,
                                    field_name=condition_field_name)

condition_combined

condition_deid = generate_query(dataset=deid,
                                person_var='deid_condition_num_persons',
                                record_var='deid_condition_num_records',
                                table_name=condition_table_name,
                                field_name=condition_field_name)

condition_deid

condition_df = create_aggregate_table_df(
    unioned=condition_unioned,
    combined=condition_combined,
    deid=condition_deid,
    unioned_persons_string='unioned_condition_num_persons',
    combined_persons_string='combined_condition_num_persons',
    deid_persons_string='deid_condition_num_persons',
    unioned_records_string='unioned_condition_num_records',
    combined_records_string='combined_condition_num_records',
    deid_records_string='deid_condition_num_records',
    person_string='Condition - Person',
    record_string='Condition - Record')

condition_df

# +
condition_df.reset_index(level=0, inplace=True)

ax = render_mpl_table(condition_df, header_columns=0, col_width=2.0)

plt.tight_layout()

plt.savefig('condition_dropoff.png', bbox_inches="tight")

# +
person_idx, record_idx = 0, 1

cp_drop = condition_df.at[person_idx, 'Total Drop (%)']
cr_drop = condition_df.at[record_idx, 'Total Drop (%)']

percent_drops['person_drop'].append(cp_drop)
percent_drops['record_drop'].append(cr_drop)
# -

# ## Looking at how volume varies in the visit_occurrence_table table

visit_table_name = 'visit_occurrence'
visit_field_name = 'visit_occurrence_id'

visit_unioned = generate_query(dataset=unioned,
                               person_var='unioned_visit_num_persons',
                               record_var='unioned_visit_num_records',
                               table_name=visit_table_name,
                               field_name=visit_field_name)

visit_unioned

visit_combined = generate_query(dataset=combined,
                                person_var='combined_visit_num_persons',
                                record_var='combined_visit_num_records',
                                table_name=visit_table_name,
                                field_name=visit_field_name)

visit_combined

visit_deid = generate_query(dataset=deid,
                            person_var='deid_visit_num_persons',
                            record_var='deid_visit_num_records',
                            table_name=visit_table_name,
                            field_name=visit_field_name)

visit_deid

visit_df = create_aggregate_table_df(
    unioned=visit_unioned,
    combined=visit_combined,
    deid=visit_deid,
    unioned_persons_string='unioned_visit_num_persons',
    combined_persons_string='combined_visit_num_persons',
    deid_persons_string='deid_visit_num_persons',
    unioned_records_string='unioned_visit_num_records',
    combined_records_string='combined_visit_num_records',
    deid_records_string='deid_visit_num_records',
    person_string='Visit - Person',
    record_string='Visit - Record')

visit_df
# +
visit_df.reset_index(level=0, inplace=True)

ax = render_mpl_table(visit_df, header_columns=0, col_width=2.0)

plt.tight_layout()

plt.savefig('visit_dropoff.jpg', bbox_inches="tight")

# +
person_idx, record_idx = 0, 1

vp_drop = visit_df.at[person_idx, 'Total Drop (%)']
vr_drop = visit_df.at[record_idx, 'Total Drop (%)']

percent_drops['person_drop'].append(vp_drop)
percent_drops['record_drop'].append(vr_drop)
# -

# ## Looking at how volume varies in the procedure_occurrence table

procedure_table_name = 'procedure_occurrence'
procedure_field_name = 'procedure_occurrence_id'

procedure_unioned = generate_query(dataset=unioned,
                                   person_var='unioned_procedure_num_persons',
                                   record_var='unioned_procedure_num_records',
                                   table_name=procedure_table_name,
                                   field_name=procedure_field_name)

procedure_unioned

procedure_combined = generate_query(dataset=combined,
                                    person_var='combined_procedure_num_persons',
                                    record_var='combined_procedure_num_records',
                                    table_name=procedure_table_name,
                                    field_name=procedure_field_name)

procedure_combined

procedure_deid = generate_query(dataset=deid,
                                person_var='deid_procedure_num_persons',
                                record_var='deid_procedure_num_records',
                                table_name=procedure_table_name,
                                field_name=procedure_field_name)

procedure_deid

procedure_df = create_aggregate_table_df(
    unioned=procedure_unioned,
    combined=procedure_combined,
    deid=procedure_deid,
    unioned_persons_string='unioned_procedure_num_persons',
    combined_persons_string='combined_procedure_num_persons',
    deid_persons_string='deid_procedure_num_persons',
    unioned_records_string='unioned_procedure_num_records',
    combined_records_string='combined_procedure_num_records',
    deid_records_string='deid_procedure_num_records',
    person_string='Procedure - Person',
    record_string='Procedure - Record')

procedure_df

# +
procedure_df.reset_index(level=0, inplace=True)

ax = render_mpl_table(procedure_df, header_columns=0, col_width=2.0)

plt.tight_layout()

plt.savefig('procedure_dropoff.jpg', bbox_inches="tight")

# +
person_idx, record_idx = 0, 1

pp_drop = procedure_df.at[person_idx, 'Total Drop (%)']
pr_drop = procedure_df.at[record_idx, 'Total Drop (%)']

percent_drops['person_drop'].append(pp_drop)
percent_drops['record_drop'].append(pr_drop)
# -

# ## Looking at how volume varies in the observation table
#
# ## NOTE: It may make sense that the combined dataset has more records than the unioned EHR dataset for the observation table. This increase is because we expect an influx of patient-provided informations from the participants' surveys.

observation_table_name = 'observation'
observation_field_name = 'observation_id'

observation_unioned = generate_query(
    dataset=unioned,
    person_var='unioned_observation_num_persons',
    record_var='unioned_observation_num_records',
    table_name=observation_table_name,
    field_name=observation_field_name)

observation_unioned

observation_combined = generate_query(
    dataset=combined,
    person_var='combined_observation_num_persons',
    record_var='combined_observation_num_records',
    table_name=observation_table_name,
    field_name=observation_field_name)

observation_combined

observation_deid = generate_query(dataset=deid,
                                  person_var='deid_observation_num_persons',
                                  record_var='deid_observation_num_records',
                                  table_name=observation_table_name,
                                  field_name=observation_field_name)

observation_deid

observation_df = create_aggregate_table_df(
    unioned=observation_unioned,
    combined=observation_combined,
    deid=observation_deid,
    unioned_persons_string='unioned_observation_num_persons',
    combined_persons_string='combined_observation_num_persons',
    deid_persons_string='deid_observation_num_persons',
    unioned_records_string='unioned_observation_num_records',
    combined_records_string='combined_observation_num_records',
    deid_records_string='deid_observation_num_records',
    person_string='Observation - Person',
    record_string='Observation - Record')

observation_df

# +
observation_df.reset_index(level=0, inplace=True)

ax = render_mpl_table(observation_df, header_columns=0, col_width=2.0)

plt.tight_layout()

plt.savefig('observation_dropoff.jpg', bbox_inches="tight")

# +
person_idx, record_idx = 0, 1

op_drop = observation_df.at[person_idx, 'Total Drop (%)']
or_drop = observation_df.at[record_idx, 'Total Drop (%)']

percent_drops['person_drop'].append(op_drop)
percent_drops['record_drop'].append(or_drop)
# -

# ## Looking at how volume varies in the drug_exposure table

drug_table_name = 'drug_exposure'
drug_field_name = 'drug_exposure_id'

drug_unioned = generate_query(dataset=unioned,
                              person_var='unioned_drug_num_persons',
                              record_var='unioned_drug_num_records',
                              table_name=drug_table_name,
                              field_name=drug_field_name)

drug_unioned

drug_combined = generate_query(dataset=combined,
                               person_var='combined_drug_num_persons',
                               record_var='combined_drug_num_records',
                               table_name=drug_table_name,
                               field_name=drug_field_name)

drug_combined

drug_deid = generate_query(dataset=deid,
                           person_var='deid_drug_num_persons',
                           record_var='deid_drug_num_records',
                           table_name=drug_table_name,
                           field_name=drug_field_name)

drug_deid

drug_df = create_aggregate_table_df(
    unioned=drug_unioned,
    combined=drug_combined,
    deid=drug_deid,
    unioned_persons_string='unioned_drug_num_persons',
    combined_persons_string='combined_drug_num_persons',
    deid_persons_string='deid_drug_num_persons',
    unioned_records_string='unioned_drug_num_records',
    combined_records_string='combined_drug_num_records',
    deid_records_string='deid_drug_num_records',
    person_string='Drug - Person',
    record_string='Drug - Record')

drug_df

# +
drug_df.reset_index(level=0, inplace=True)

ax = render_mpl_table(drug_df, header_columns=0, col_width=2.0)

plt.tight_layout()

plt.savefig('drug_dropoff.jpg', bbox_inches="tight")
# +
person_idx, record_idx = 0, 1

dp_drop = drug_df.at[person_idx, 'Total Drop (%)']
dr_drop = drug_df.at[record_idx, 'Total Drop (%)']

percent_drops['person_drop'].append(dp_drop)
percent_drops['record_drop'].append(dr_drop)
# -

# ## Not let's put all of the percent drops into a single dataframe

# +
person_drops = percent_drops['person_drop']
record_drops = percent_drops['record_drop']
final_dict = {}

for table_name, person_drop, record_drop in zip(table_order, person_drops,
                                                record_drops):
    final_dict[table_name] = [person_drop, record_drop]
# -

overall_drop_df = pd.DataFrame(data=final_dict,
                               index=['Person Drop (%)', 'Record Drop (%)'])

# +
overall_drop_df.reset_index(level=0, inplace=True)

ax = render_mpl_table(overall_drop_df, header_columns=0, col_width=2.0)

plt.tight_layout()

plt.savefig('overall_percent_dropoff.jpg', bbox_inches="tight")

# -

# ## Now we are going to look at the dropoff across the different sites

def generate_site_level_query(id_name, unioned, table_name, combined):
    """
    Function is used to generate the dataframe that shows the following items:
        a. each source HPO ID
        b. the number of rows for the HPO for a particular table in the unioned dataset
        c. the number of rows for the HPO for a particular table in the combined dataset
        d. the total 'drop' of rows across unioned to combined, expressed as a percentage

    Parameters
    ----------
    id_name (string): represents the 'primary key' of the table (the unique identifier
                      for each row)

    unioned (string): the name of the unioned dataset to be queried

    table_name (string): name of the table that is being investigated

    combined (string): the name of the combined dataset to be queried

    Returns
    -------
    dataframe (df): contains all of the information outlined in the top of the docstring
    """

    site_level_query = """
    SELECT
      DISTINCT a.src_hpo_id AS source_hpo,
      a.num_rows_unioned,
      b.num_rows_combined,
      ROUND( (a.num_rows_unioned - b.num_rows_combined) / a.num_rows_unioned * 100, 2) AS percent_unioned_rows_dropped
    FROM (
      SELECT
        DISTINCT mx.src_hpo_id,
        COUNT(x1.{id_name}) AS num_rows_unioned,
      FROM
        `{unioned}.{table_name}` x1
      LEFT JOIN
        `{unioned}._mapping_{table_name}` mx
      ON
        x1.{id_name} = mx.{id_name}
      GROUP BY
        1) a
    JOIN (
      SELECT
        DISTINCT mx.src_hpo_id,
        COUNT(x2.{id_name}) AS num_rows_combined,
      FROM
        `{combined}.{table_name}` x2
      LEFT JOIN
        `{combined}._mapping_{table_name}` mx
      ON
        x2.{id_name} = mx.{id_name}
      GROUP BY
        1 ) b
    ON
      a.src_hpo_id = b.src_hpo_id
    ORDER BY
      percent_unioned_rows_dropped DESC
    """.format(id_name=id_name,
               unioned=unioned,
               table_name=table_name,
               combined=combined)

    dataframe = utils.bq.query(site_level_query)

    return dataframe

def add_total_drop_row(dataframe):
    """
    Function is used to add a 'total' row at the bottom of a dataframe that shows the
    relative 'drop' across the pipeline (unioned to combined) for the different sites.

    This row will show:
        a. the number of rows in the unioned dataset
        b. the number of rows in the combined dataset
        c. the total 'drop' of rows across unioned to combined, expressed as a percentage

    Parameters:
    ----------
    dataframe (df): dataframe for a particular table. shows a-c (above) for each of the
        HPOs that uploaded data

    Returns:
    --------
    dataframe (df): the inputted dataframe with an additional 'total' row at the end
    """

    dataframe = dataframe.append(
        dataframe.sum(numeric_only=True).rename('Total'))

    hpo_names = dataframe['source_hpo'].tolist()

    hpo_names[-1:] = ["Total"]

    dataframe['source_hpo'] = hpo_names

    unioned_total = dataframe.loc['Total']['num_rows_unioned']

    combined_total = dataframe.loc['Total']['num_rows_combined']

    total_drop_percent = round(
        (unioned_total - combined_total) / unioned_total * 100, 2)

    dataframe.at['Total', 'percent_unioned_rows_dropped'] = total_drop_percent

    return dataframe

def add_percent_of_drop_column(dataframe):
    """
    Function is used to add a 'percent_of_drop' column that shows how much
    each site's 'drop' contributed to the 'overall' drop from the unioned
    to the combined steps of the pipeline.

    Parameters
    ----------
    dataframe (df): dataframe for a particular table. shows the following for
                    HPOs that uploaded data:

        a. the number of rows in the unioned dataset
        b. the number of rows in the combined dataset
        c. the total 'drop' of rows across unioned to combined, expressed as a percentage

    Returns
    -------
    dataframe (df): the above dataframe with a new column that shows each site's
                    'contribution' to the overall drop between unioned and
                    combined
    """
    unioned_total = dataframe.loc['Total']['num_rows_unioned']
    combined_total = dataframe.loc['Total']['num_rows_combined']
    total_drop = unioned_total - combined_total

    hpo_list = dataframe['source_hpo'].tolist()

    rows_unioned = dataframe['num_rows_unioned']
    rows_combined = dataframe['num_rows_combined']

    subtracted = map(operator.sub, rows_unioned, rows_combined)
    subtracted_list = list(subtracted)

    percent_of_drop = [round(x / total_drop * 100, 2) for x in subtracted_list]

    dataframe['percent_of_drop'] = percent_of_drop

    return dataframe

# ### Lets see the dropoff in the measurement table

measurement_info = generate_site_level_query(id_name='measurement_id',
                                             unioned=unioned,
                                             table_name='measurement',
                                             combined=combined)

measurement_info = add_total_drop_row(measurement_info)

measurement_info = add_percent_of_drop_column(measurement_info)

# +
measurement_info_dict = create_dicts_w_info(
    df=measurement_info,
    x_label='source_hpo',
    column_label='percent_unioned_rows_dropped')

create_graphs(info_dict=measurement_info_dict,
              xlabel='HPO',
              ylabel='% Rows Dropped from Unioned to Combined',
              title='Measurement Drop (Unioned to Combined) By Site',
              img_name='unioned_combined_measurement_drop.jpg',
              color='b',
              total_diff_color=True,
              turnoff_x=False)
# -

measurement_info = measurement_info.sort_values(by='percent_of_drop',
                                                ascending=False)

create_pie_chart(
    measurement_info,
    title='Unioned to Combined Drop \n (Measurement) Contributions',
    img_name='measurement_unioned_combined_drop_site_contribution.jpg')

# ### Drug Exposure Table

drug_info = generate_site_level_query(id_name='drug_exposure_id',
                                      unioned=unioned,
                                      table_name='drug_exposure',
                                      combined=combined)

drug_info = add_total_drop_row(drug_info)

drug_info = add_percent_of_drop_column(drug_info)

# +
drug_info_dict = create_dicts_w_info(
    df=drug_info,
    x_label='source_hpo',
    column_label='percent_unioned_rows_dropped')

create_graphs(info_dict=drug_info_dict,
              xlabel='HPO',
              ylabel='% Rows Dropped from Unioned to Combined',
              title='Drug Drop (Unioned to Combined) By Site',
              img_name='unioned_combined_drug_drop.jpg',
              color='b',
              total_diff_color=True,
              turnoff_x=False)
# -

drug_info = drug_info.sort_values(by='percent_of_drop', ascending=False)

create_pie_chart(drug_info,
                 title='Unioned to Combined Drop \n (Drug) Contributions',
                 img_name='drug_unioned_combined_drop_site_contribution.jpg')

# ### Procedure Occurrence Table

procedure_info = generate_site_level_query(id_name='procedure_occurrence_id',
                                           unioned=unioned,
                                           table_name='procedure_occurrence',
                                           combined=combined)

procedure_info = add_total_drop_row(procedure_info)

procedure_info = add_percent_of_drop_column(procedure_info)

# +
procedure_info_dict = create_dicts_w_info(
    df=procedure_info,
    x_label='source_hpo',
    column_label='percent_unioned_rows_dropped')

create_graphs(info_dict=procedure_info_dict,
              xlabel='HPO',
              ylabel='% Rows Dropped from Unioned to Combined',
              title='Procedure Drop (Unioned to Combined) By Site',
              img_name='unioned_combined_procedure_drop.jpg',
              color='b',
              total_diff_color=True,
              turnoff_x=False)
# -

procedure_info = procedure_info.sort_values(by='percent_of_drop',
                                            ascending=False)

create_pie_chart(
    procedure_info,
    title='Unioned to Combined Drop \n (Procedure) Contributions',
    img_name='procedure_unioned_combined_drop_site_contribution.jpg')

# ### Visit Occurrence Table

visit_info = generate_site_level_query(id_name='visit_occurrence_id',
                                       unioned=unioned,
                                       table_name='visit_occurrence',
                                       combined=combined)

visit_info = add_total_drop_row(visit_info)

visit_info = add_percent_of_drop_column(visit_info)

# +
visit_info_dict = create_dicts_w_info(
    df=visit_info,
    x_label='source_hpo',
    column_label='percent_unioned_rows_dropped')

create_graphs(info_dict=visit_info_dict,
              xlabel='HPO',
              ylabel='% Rows Dropped from Unioned to Combined',
              title='Visit Drop (Unioned to Combined) By Site',
              img_name='unioned_combined_visit_drop.jpg',
              color='b',
              total_diff_color=True,
              turnoff_x=False)

# +
visit_info = visit_info.sort_values(by='percent_of_drop', ascending=False)

create_pie_chart(visit_info,
                 title='Unioned to Combined Drop \n (Visit) Contributions',
                 img_name='visit_unioned_combined_drop_site_contribution.jpg')
# -

# ### Observation Table

observation_info = generate_site_level_query(id_name='observation_id',
                                             unioned=unioned,
                                             table_name='observation',
                                             combined=combined)

# +
observation_info = add_total_drop_row(observation_info)

observation_info = add_percent_of_drop_column(observation_info)

# +
observation_info_dict = create_dicts_w_info(
    df=observation_info,
    x_label='source_hpo',
    column_label='percent_unioned_rows_dropped')

create_graphs(info_dict=observation_info_dict,
              xlabel='HPO',
              ylabel='% Rows Dropped from Unioned to Combined',
              title='Observation Drop (Unioned to Combined) By Site',
              img_name='unioned_combined_observation_drop.jpg',
              color='b',
              total_diff_color=True,
              turnoff_x=False)

# +
observation_info = observation_info.sort_values(by='percent_of_drop',
                                                ascending=False)

create_pie_chart(
    observation_info,
    title='Unioned to Combined Drop \n (Observation) Contributions',
    img_name='observation_unioned_combined_drop_site_contribution.jpg')


from collections import defaultdict

class Solution:
    def minDistance(self, word1: str, word2: str) -> int:

        F = defaultdict(lambda: defaultdict(int))
        m = len(word1)
        n = len(word2)

        for i in range(1, m + 1):
            for j in range(1, n + 1):
                if word1[i-1] == word2[j-1]:
                    F[i][j] = F[i-1][j-1] + 1
                else:
                    F[i][j] = max(
                        F[i-1][j],
                        F[i][j-1],
                    )

        return m - F[m][n] + n - F[m][n]

    def minDistance_edit_distance(self, word1: str, word2: str) -> int:

        F = defaultdict(lambda: defaultdict(int))
        m = len(word1)
        n = len(word2)

        for i in range(1, m + 1):
            F[i][0] = i
        for j in range(1, n + 1):
            F[0][j] = j

        for i in range(1, m + 1):
            for j in range(1, n + 1):
                if word1[i-1] == word2[j-1]:
                    F[i][j] = F[i-1][j-1]
                else:
                    F[i][j] = min(
                        F[i-1][j] + 1,
                        F[i][j-1] + 1,
                    )

        return F[m][n]

if __name__ == "__main__":
    assert Solution().minDistance("sea", "eat") == 2

data_files/583 Delete Operation for Two Strings.py


import pandas as pd
from shapely.geometry import Point
import geopandas as gpd

import matplotlib.pyplot as plt

plt.rcParams.update({'font.size': 20})

geometry = [Point(xy) for xy in zip(tmp['LONGNUM'], tmp['LATNUM'])]
gdf = gpd.GeoDataFrame(tmp, geometry=geometry)

#this is a simple map that goes with geopandas
world = gpd.read_file(gpd.datasets.get_path('naturalearth_lowres'))
africa = world.query('continent == "Africa"')
gdf.plot(ax=africa.plot(figsize=(20, 12), edgecolor='gray', color='#FFFDD0'), column = "Under5_Mortality_Rate", marker='o', markersize=3, cmap = "inferno_r", legend=True,
         legend_kwds={"label":"Under 5 Mortality Rate"})

plt.axis('off')
plt.show()

geometry = [Point(xy) for xy in zip(df['LONGNUM'], df['LATNUM'])]
gdf = gpd.GeoDataFrame(df, geometry=geometry)
gdf["Mean_BMI"] = np.where(gdf["Mean_BMI"]> 35, 35, gdf["Mean_BMI"])

#this is a simple map that goes with geopandas
world = gpd.read_file(gpd.datasets.get_path('naturalearth_lowres'))
africa = world.query('continent == "Africa"')
gdf.plot(ax=africa.plot(figsize=(20, 12), edgecolor='gray', color='#FFFDD0'), column = "Mean_BMI", marker='o', markersize=3, cmap = "inferno_r", legend=True,
         legend_kwds={"label":"Mean Body Mass Index"})

plt.axis('off')
plt.show()


#!/usr/bin/env python3

import os
import sys
import re

import json

from vmaf.core.result import Result

__copyright__ = "Copyright 2016-2020, Netflix, Inc."
__license__ = "BSD+Patent"

def print_usage():
    print("usage: python " + os.path.basename(sys.argv[0]) + " input_file\n")
    print("input_file contains a list of files for assembly (can be xml or json)")

class FileAssembler:

    SUPPORTED_FILE_TYPES = ['xml', 'json']

    def __init__(self, to_assemble_input):
        self.to_assemble_input = to_assemble_input

    @staticmethod
    def create_assembly_file_list(to_assemble_input):

        to_assemble_list = []
        if isinstance(to_assemble_input, list):
            to_assemble_list = to_assemble_input
        else:
            with open(to_assemble_input, "rt") as input_file:
                for line in input_file.readlines():

                    # match comment
                    mo = re.match(r"^#", line)
                    if mo:
                        print("Skip commented line: {}".format(line))
                        continue

                    # match whitespace
                    mo = re.match(r"[\s]+", line)
                    if mo:
                        continue

                    mo = re.match(r"([\S]+)", line)
                    if not mo:
                        print("Invalid file: {}".format(line))
                        print_usage()
                        return 1

                    to_assemble_list.append(line.strip())

        return to_assemble_list

    def _create_result_list(self, to_assemble_list):
        pass

    def assemble(self):
        """
        Main file assembly logic
        """
        to_assemble_list = self.create_assembly_file_list(self.to_assemble_input)
        self._assert(to_assemble_list)
        results = self._create_result_list(to_assemble_list)
        combined_result = Result.combine_result(results)

        return combined_result

    def _assert(self, to_assemble_list):
        """
        Perform necessary assertions before parsing any of the files.
        """

        # check that the number of files is greater than 0
        assert len(to_assemble_list) > 0
        # check that the file formats match
        assemble_format_list = [os.path.splitext(f)[1].split(".")[1] for f in to_assemble_list]
        assert len(set(assemble_format_list)) == 1, "The file formats for assembly do not much."
        # check that the file format is supported for assembly
        assert assemble_format_list[0] in self.SUPPORTED_FILE_TYPES, \
            "The assembly format is not consistent, use any of {fmts}".format(fmts=str(self.SUPPORTED_FILE_TYPES))

class XmlAssembler(FileAssembler):

    @staticmethod
    def _parse_files(to_assemble_list):

        to_assemble_xml_strings = []
        for to_assemble_xml in to_assemble_list:
            with open(to_assemble_xml, 'r') as f:
                to_assemble_xml_strings.append(f.read())

        return to_assemble_xml_strings

    def _create_result_list(self, to_assemble_list):

        to_assemble_xml_strings = self._parse_files(to_assemble_list)

        results = []
        for to_assemble_xml_string in to_assemble_xml_strings:
            results.append(Result.from_xml(to_assemble_xml_string))

        return results

class JsonAssembler(FileAssembler):

    @staticmethod
    def _parse_files(to_assemble_list):

        to_assemble_json_strings = []
        for json_file in to_assemble_list:
            with open(json_file, 'r') as f:
                to_assemble_json_strings.append(json.load(f))

        return to_assemble_json_strings

    def _create_result_list(self, to_assemble_list):

        to_assemble_jsons = self._parse_files(to_assemble_list)

        results = []
        for to_assemble_json in to_assemble_jsons:
            to_assemble_json_string = json.dumps(to_assemble_json)
            results.append(Result.from_json(to_assemble_json_string))

        return results

def main():

    if len(sys.argv) != 2:
        print_usage()
        return 2

    files_to_assemble_list = sys.argv[1]

    desired_file_list = FileAssembler.create_assembly_file_list(files_to_assemble_list)

    if ".xml" in desired_file_list[0]:
        xml_output = XmlAssembler(files_to_assemble_list).assemble().to_xml()
        print(xml_output)
    elif ".json" in desired_file_list[0]:
        json_output = JsonAssembler(files_to_assemble_list).assemble().to_json()
        print(json_output)
    else:
        print_usage()
        return 2

if __name__ == "__main__":
    ret = main()
    exit(ret)


import csv
import pandas as pd
from tokenizations import get_alignments
import torch

pd.set_option('mode.chained_assignment', None)

def get_texts(corpus):
	'''
	Get a list of texts in the corpus
	'''
	texts = []
	text_ids = pd.unique(corpus.speechId)
	for t in text_ids:
		text = corpus[corpus.speechId==t]
		texts.append(text)
	return texts

def split_text(text, max_words=700, min_words=50):
	'''
	LM can't take too many tokens at once. So less than 700 words are fed into
	the model each time.
	'''
	result = []
	paragraph_ids = pd.unique(text.paragraphId)
	df = pd.DataFrame()
	for p in paragraph_ids:
		paragraph = text[text.paragraphId==p]
		if df.shape[0] + paragraph.shape[0] <= max_words:
			df = pd.concat([df, paragraph])
		else:
			result.append(df)
			df = paragraph
	# deal with the last batch of subtext
	if df.shape[0] <= min_words:
		result[-1] = pd.concat([result[-1], df])
	else:
		result.append(df)

	return result

def get_utt_logp(utterance, tokenizer, model):
	'''
	Given utterance input, return Transformer results as a list of
	tuples (token, logprob).
	'''
	result = []
	tokens = tokenizer.encode(utterance)
	lm_output = model(torch.tensor(tokens))
	lm_logp = torch.log_softmax(lm_output.logits, -1)
	for idx, t in enumerate(tokens):
		token = tokenizer.decode(t).strip()
		if idx == 0:
			logprob = None
		else:
			logprob = lm_logp[idx-1, t].item()
		result.append((token, logprob))
	return result

def add_text_logp(text, tokenizer, model):
	'''
	Given a corpus text, get the logp for each word and add one column of logp
	to the original text df. No return.
	'''
	tokens_corpus = text.word.tolist()
	utt = ' '.join(tokens_corpus)
	model_output = get_utt_logp(utt, tokenizer, model)
	num_lm_tokens = len(model_output)
	aligned_logp = align_tokenization(tokens_corpus, model_output)
	text['logp'], text['lm_tokens'] = aligned_logp, num_lm_tokens

def align_tokenization(tokens_corpus, model_output):
	'''
	Align the tokenization between model output and the corpus. Specifically,
	we want the tokenization in the corpus, but align it with model output.
	'''
	result = []
	tokens_lm = [w[0] for w in model_output]
	logprobs = [w[1] for w in model_output]
	corpus2lm, _ = get_alignments(tokens_corpus, tokens_lm)
	for i in corpus2lm:
		if len(i) == 1:
			logp = logprobs[i[0]]
		elif len(i) == 0:    # irreconcilable alignment
			logp = None
		else:
			if logprobs[i[0]] == None:
				logp = None
			else:
				logp = sum(logprobs[i[0]:i[-1]+1])
		result.append(logp)
	return result

def main(RT_corpus, output_file, tokenizer, model, if_test=True):
	'''
	Given an RT corpus, write a csv file that includes a column of log probability
	aligned with each token in the RT corpus.
	'''
	danish = pd.read_csv(RT_corpus)
	texts = get_texts(danish)
	if if_test:
		texts = texts[:1]   # Only use the first two texts if testing
	df = pd.DataFrame()
	for text in texts:
		subtexts = split_text(text)
		for idx, subtext in enumerate(subtexts):
			add_text_logp(subtext, tokenizer, model)
			subtext['subtextId'] = idx
			subtext['subtext_wordId'] = subtext.groupby('subtextId').cumcount()
			df = pd.concat([df, subtext])
	df.to_csv(output_file, index=False)


#!/usr/bin/env python3

import os
import urllib.request
import requests
import pygal

from bs4 import BeautifulSoup
from collections import Counter
from pygal.style import Style

from flask import Blueprint,render_template,request,redirect,url_for,session,flash
from time import gmtime, strftime

from ..models.model import User, ShoeBox, Sneaker

elekid = Blueprint('private',__name__,url_prefix='/2492')

def tsplit(string, delimiters):
    """Behaves str.split but supports multiple delimiters."""

    delimiters = tuple(delimiters)
    stack = [string,]

    for delimiter in delimiters:
        for i, substring in enumerate(stack):
            substack = substring.split(delimiter)
            stack.pop(i)
            for j, _substring in enumerate(substack):
                stack.insert(i+j, _substring)
    return stack

def add_dict_total(dict, val):
    """VAL is a parameter to search for price_bought,
       value and profit which are values in shoebox dict"""
    sum_list = []
    for key in dict.keys():
        if val == 'value':
            if dict[key]['type'] != 'SELL':
                sum_list.append(dict[key][val])
        else:
            sum_list.append(dict[key][val])
    return sum(sum_list)

def scrape_new_shoe(url):
    try:
        """CONTAINERS"""
        shoe_html = requests.get(url).content
        shoe_soup = BeautifulSoup(shoe_html, 'html.parser')
        shoe_container = shoe_soup.find("div", {"class": "product-view"})
        header_stat = shoe_container.find_all('div', {'class': 'header-stat'})
        """HEADER INFO"""
        name = shoe_container.find('h1').get_text().replace('/','-').strip()
        new_name = name.replace('?','')
        image = shoe_container.find('div', {'class': 'product-media'}).img['src']
        ticker = header_stat[1].get_text().strip().split(' ')[1]
        """PRODUCT INFO"""
        product_info = shoe_container.find('div', {'class': 'product-info'}).get_text().strip()
        product_data = tsplit(product_info,('Style ',' Colorway ',' Retail Price ', ' Release Date '))
        colorway = product_data[2]
        retail_price = product_data[3]
        release_date = product_data[4][:10]
        """MARKET INFO"""
        market_summary = shoe_container.find('div', {'class': 'product-market-summary'}).get_text().strip()
        market_data = tsplit(market_summary,('52 Week High ',' | Low ','Trade Range (12 Mos.)','Volatility'))
        year_high = market_data[1]
        year_low = market_data[2]
        trade_range = market_data[3]
        """HISTORICAL"""
        twelve_month_historical = shoe_container.find('div', {'class': 'gauges'}).get_text().strip()
        twelve_data = tsplit(twelve_month_historical,('# of Sales','Price Premium(Over Original Retail Price)','Average Sale Price'))
        total_sales = twelve_data[1]
        avg_sale_price = twelve_data[3]
        """BRAND INFO"""
        trail = shoe_container.find('div', {'class': 'grails-crumbs'}).get_text()
        identifier = trail[12:13]

        if identifier == 'O':
            brand = 'Other'
        elif identifier == 'a':
            brand = 'Adidas'
        elif identifier == 'N':
            brand = 'Nike'
        elif identifier == 'J':
            brand = 'Jordan'
        else:
            brand = 'Other'

        new_totalSales = total_sales.replace(',','')

        retailPrice = retail_price.strip('$')
        new_retailPrice = retailPrice.replace(',','')

        avgSalePrice = avg_sale_price.strip('$')
        new_avgSalePrice = avgSalePrice.replace(',','')

        yearHigh = year_high.strip('$')
        new_yearHigh = yearHigh.replace(',','')

        yearLow = year_low.strip('$')
        new_yearLow = yearLow.replace(',','')

        if 'Harden' in name.split(' ') and brand != 'Nike':
            type = 'Harden'
        elif 'Curry' in name.split(' ') and brand != 'Nike':
            type = 'Curry'
        elif 'PG' in name.split(' '):
            type = 'PG'
        elif 'Westbrook' in name.split(' '):
            type = 'Westbrook'
        elif 'Kyrie' in name.split(' '):
            type = 'Kyrie'
        elif 'Dame' in name.split(' '):
            type = 'Dame'
        elif 'React' in name.split(' '):
            type = 'React'
        elif 'Foamposite' in name.split(' '):
            type = 'Foamposite'
        elif 'NMD' in name.split(' '):
            type = 'NMD'
        elif 'Ultra Boost' in name.split(' ') or 'UltraBoost' in name.split(' '):
            type = 'Ultraboost'
        elif 'Air Force' in name.split(' '):
            type = 'Air Force'
        elif 'Air Max' in name.split(' '):
            type = 'Air Max'
        elif 'SB' in name.split(' '):
            type = 'SB'
        elif 'React' in name.split(' '):
            type = 'React'
        elif 'Foamposite' in name.split(' '):
            type = 'Foamposite'
        elif 'KD' in name.split(' '):
            type = 'KD'
        elif 'Lebron' in name.split(' ') or 'UltraBoost' in name.split(' '):
            type = 'Lebron'
        elif 'Kobe' in name.split(' '):
            type = 'Kobe'
        elif 'Yeezy' in name.split(' '):
            type = 'Yeezy'
        elif 'Jordan' in name.split(' ') and '1' in name.split(' '):
            type = '1'
        elif 'Jordan' in name.split(' ') and '2' in name.split(' '):
            type = '2'
        elif 'Jordan' in name.split(' ') and '3' in name.split(' '):
            type = '3'
        elif 'Jordan' in name.split(' ') and '4' in name.split(' '):
            type = '4'
        elif 'Jordan' in name.split(' ') and '5' in name.split(' '):
            type = '5'
        elif 'Jordan' in name.split(' ') and '6' in name.split(' '):
            type = '6'
        else:
            type = 'Other'

        if new_avgSalePrice == '--' or new_retailPrice =='--':
            premium = None
        else:
            value = (float(new_avgSalePrice)-float(new_retailPrice))/float(new_retailPrice)
            premium = '{:.2f}'.format(value*100)

        data = {
                "name" : new_name,
                "url" : url,
                "brand": brand,
                "type": type,
                "image" : image,
                "image_placeholder" : '--',
                "ticker" : ticker,
                "colorway" : colorway,
                "retail_price" : new_retailPrice,
                "release_date" : release_date,
                "year_high" : new_yearHigh,
                "year_low" : new_yearLow,
                "trade_range" : trade_range,
                "total_sales" : new_totalSales,
                "avg_sale_price" : new_avgSalePrice,
                "premium" : premium
            }

    except AttributeError:
        print('attribute error trying again')
        scrape_new_shoe(url)

    except IndexError:
        print('index error, try a valid url')
        data = {}

    return data

def insert_shoe_to_db(data):
    sneaker = Sneaker()
    sneaker.brand	          = data['brand']
    sneaker.type              = data['type']
    sneaker.name              = data['name']
    sneaker.colorway          = data['colorway']
    sneaker.image             = data['image']
    sneaker.image_placeholder = data['image_placeholder']
    sneaker.release_date      = data['release_date']
    sneaker.retail_price      = data['retail_price']
    sneaker.ticker            = data['ticker']
    sneaker.total_sales       = data['total_sales']
    sneaker.url               = data['url']
    sneaker.year_high         = data['year_high']
    sneaker.year_low          = data['year_low']
    sneaker.avg_sale_price    = data['avg_sale_price']
    sneaker.premium           = data['premium']
    sneaker.save(data['name'])

def download_sneaker_img(data):
    img_link = data['image']
    print(img_link)
    name = data['name']

    if 'Placeholder' in img_link.split('-'):

        placeholder = open('/Users/ahn.ch/Desktop/sb_placeholder.jpg')
        f = open('run/src/static/{}.jpg'.format(name),'wb')
        f.write(placeholder.read())
        f.close()

        print('Placeholder Added')

    else:
        picture_request = requests.get(img_link)
        if picture_request.status_code == 200:
            with open("run/src/static/{}.jpg".format(name), 'wb') as f:
                f.write(picture_request.content)

def account_pairing_scores(pk):

    user = User()
    account_preferences = user.get_account_preferences(pk)
    other_preferences = user.get_other_account_preferences(pk)

    print(account_preferences)
    print(other_preferences)

    pairing_scores = []
    for key,value in account_preferences.items():

        user_brand = account_preferences[key]['brand']
        user_color = account_preferences[key]['color']

        for key, value in other_preferences.items():

            brand_score, color_score = 0, 0

            for brand in other_preferences[key]['brand']:
                if brand in user_brand:
                    brand_score += 18
            for color in other_preferences[key]['color']:
                if color in user_color:
                    color_score += 12

            pairing_scores.append((key,(brand_score+color_score)))

    sorted_pairing_scores = sorted(pairing_scores, key=lambda x:x[1])[::-1]
    pk1 = sorted_pairing_scores[0][0]
    pk2 = sorted_pairing_scores[1][0]
    return [pk1,pk2]

def shoebox_graph_labels(box, val):
    date_labels = []
    num_labels = []
    values = []

    for key in box.keys():
        if box[key]['date'] not in date_labels:
            date_labels.append(box[key]['date'])
    if val == 'value':
        for key in box.keys():
            for i in range(len(date_labels)):
                if box[key]['date'] == date_labels[i]:
                    if box[key]['type'] != 'SELL':
                        num_labels.append({box[key]['date']: box[key]['{}'.format(val)]})
                    else:
                        num_labels.append({box[key]['date']: 0 })
                else:
                    pass
    elif val == 'profit':
        for key in box.keys():
            for i in range(len(date_labels)):
                if box[key]['date'] == date_labels[i]:
                    num_labels.append({box[key]['date']: box[key]['{}'.format(val)]})
                else:
                    pass

    print(date_labels)
    print(num_labels)
    print(values)
    c = Counter()
    for d in num_labels:
        c.update(d)

    result = [{key: value} for key, value in c.items()]

    for vals in result:
        for key in vals.keys():
            values.append(vals[key])

    final = [sum(values[:i+1]) for i in range(len(values))]
    print(final)

    return [date_labels,final]

@elekid.route('/account',methods=['GET','POST'])
def account():
    if request.method == 'GET':
        try:
            """
            ~~~~~~~~~
            USER INFO
            ~~~~~~~~~
            """
            user    = User({'username': session['username'],
                            'pk': session['pk'], 'age': session['age'],
                            'gender': session['gender']})
            favList = user.display_favorites()
            shoebox = user.display_shoebox()
            spent   = add_dict_total(shoebox, 'price_bought')
            worth   = add_dict_total(shoebox, 'value')
            profit  = add_dict_total(shoebox, 'profit')

            """
            ~~~~~~~~~~~
            PyGAL CHART
            ~~~~~~~~~~~
            """
            label_list = shoebox_graph_labels(shoebox, 'value')
            profit_list = shoebox_graph_labels(shoebox, 'profit')

            date_label = label_list[0]
            value_label = label_list[1]
            profit_label = profit_list[1]

            print(value_label)
            print(profit_label)

            custom_style = Style(
                colors = ('purple')
            )

            line_chart = pygal.Line(fill=True,explicit_size=True, height=500,width=500,style=custom_style)
            line_chart.x_labels = date_label
            line_chart.add('Box Value', value_label)
            line_chart.render()

            line_chart_profit = pygal.Line(fill=True, explicit_size=True, height=500,width=500)
            line_chart_profit.x_labels = date_label
            line_chart_profit.add('Profit', profit_label)
            line_chart_profit.render()

            return render_template('private/account.html',
                                    favList = favList,
                                    shoebox=shoebox,
                                    profit  = profit,
                                    spent   = spent,
                                    worth   = worth,
                                    chart   = line_chart,
                                    chart2  = line_chart_profit )
        except KeyError:
            return redirect('/register')
    elif request.method == 'POST':
        if request.form['post_button'].split('-')[0] == 'Update':
            box_pk   = request.form['post_button'].split('-')[1]
            box      = ShoeBox(pk=box_pk)
            shoeName = box.shoename
            return redirect('/update-box/'+box_pk+'/'+shoeName)
        elif request.form['post_button'].split('-')[0] == 'Remove':
            box_pk = request.form['post_button'].split('-')[1]
            box = ShoeBox(pk=box_pk)
            box.remove(box_pk)
            user = User({'username': session['username'],
                         'pk': session['pk'], 'age': session['age'],
                         'gender': session['gender']})
            favList = user.display_favorites()
            shoebox = user.display_shoebox()
            spent   = add_dict_total(shoebox, 'price_bought')
            worth   = add_dict_total(shoebox, 'value')
            profit  = add_dict_total(shoebox, 'profit')
            label_list = shoebox_graph_labels(shoebox, 'value')
            profit_list = shoebox_graph_labels(shoebox, 'profit')

            date_label = label_list[0]
            value_label = label_list[1]
            profit_label = profit_list[1]

            print(value_label)
            print(profit_label)

            line_chart = pygal.Line(fill=True,explicit_size=True, height=500,width=500)
            line_chart.x_labels = date_label
            line_chart.add('Box Value', value_label)
            line_chart.render()

            line_chart_profit = pygal.Line(fill=True,explicit_size=True, height=500,width=500)
            line_chart_profit.x_labels = date_label
            line_chart_profit.add('Profit', profit_label)
            line_chart_profit.render()
            return render_template('private/account.html',
                                    favList = favList,shoebox=shoebox,
                                    profit  = profit,
                                    spent   = spent,
                                    worth   = worth,
                                    chart   = line_chart,
                                    chart2  = line_chart_profit )
    else:
        pass

@elekid.route('/favorites',methods=['GET','POST'])
def favorites():
    if request.method == 'GET':
        try:
            """
            ~~~~~~~~~
            USER INFO
            ~~~~~~~~~
            """
            user    = User({'username': session['username'],
                            'pk': session['pk'], 'age': session['age'],
                            'gender': session['gender']})
            favList = user.display_favorites()

            like_account = account_pairing_scores(user.pk)
            pk1 = like_account[0]
            pk2 = like_account[1]
            print(pk1)

            u = User()
            recList = u.display_reccomendations(pk1,pk2)

            print(recList)

            return render_template('private/favorites.html',
                                    favList = favList,
                                    recList = recList )
        except KeyError:
            return redirect('/register')
    elif request.method == 'POST':
        pass
    else:
        pass

@elekid.route('/upload',methods=['GET','POST'])
def upload():
    if request.method == 'GET':
        return render_template('/private/upload.html')
    elif request.method == 'POST':
        name = request.form['url']
        print(name)
        lower_name = name.lower().split(' ')
        print(lower_name)
        join_name = '-'.join(lower_name)
        print(join_name)
        url = 'https://stockx.com/'+str(join_name)
        print(url)
        sneaker = Sneaker()
        shoeData = scrape_new_shoe(url)
        if not sneaker.existing(shoeData['name']):
            download_sneaker_img(shoeData)
            print('.')
            print('.')
            print('.')
            print('DOWNLOADED IMAGE: {}'.format(shoeData['image']))
            print('.')
            print('.')
            print('.')
            print('.')
            print('.')
            print('.')
            insert_shoe_to_db(shoeData)
            print('ADDED SHOE TO DATABASE - CONGRATS DUDE')
            print('.')
            print('.')
            print('.')
            print('.')
            print('.')
            print('.')
            return redirect('/add/success')
        else:
            return render_template('/private/upload.html',
                                message='Shoe Exists Homie!')
    else:
        pass


import numpy as np
from matplotlib import image
import matplotlib.pyplot as plt
from tqdm import tqdm

def calculate_k_means(data, k):
    n, d = data.shape

    # random initialize means
    mew_old = mew = np.random.randn(k, d)

    # repeatedly perform assignment and mean update
    # hard limit is 50
    print('starting k-means')
    for x in tqdm(range(50)):

        # assignment
        assignment = np.argmin(np.asarray([np.sum((data - mew[a]) ** 2, axis=1) for a in range(k)]), axis=0)

        # mean update
        mew = np.asarray([np.sum((assignment == a) * (data.transpose()), axis=1) / (
            np.sum(assignment == a) + 1e-8) for a in range(k)])

        # until convergence
        if np.sum((mew - mew_old) ** 2) < 1e-10:
            break
        mew_old = mew

    print("converged at iteration: ", x)

    return mew

def k_means_clustering(data, k):
    # find out k means
    mew = calculate_k_means(data, k)
    print("converged mew: \n", mew)

    # assign clusters
    assignment = np.argmin(np.asarray([np.sum((data - mew[a]) ** 2, axis=1) for a in range(k)]), axis=0)

    return assignment

def main():
    img_mat = image.imread('images/cow.jpg')
    h, w, d = img_mat.shape
    k = 2
    img_mat = img_mat.reshape((h * w, d))
    assignment = k_means_clustering(img_mat, k).reshape(h, w)

    # duplicate to create image to RGB
    assignment = np.asarray(np.dstack((assignment, assignment, assignment)), dtype=np.float32) / (k - 1 + 1e-20)
    plt.imshow(assignment)
    plt.show()

if __name__ == '__main__':
    main()


from typing import TypeVar

INITIAL_SIZE = 0
MIN_INITIAL_CAPACITY = 1
MIN_REAL_CAPACITY = 16
GROWTH_FACTOR = 2
SHRINK_FACTOR = 4

INDEX_OUT_OF_RANGE_ERROR = "Index out of range."
NOT_MATCH_VALUE_FOUND_ERROR = "No match value found."

T = TypeVar("T", int, str, None)

class Array:
    def __init__(self, capacity: int = MIN_INITIAL_CAPACITY) -> None:
        """
        Creates a new array according to the provided capacity.

        About: An array is a contiguous area of memory consisting of equal-size
        elements indexed by contiguous integers.

        - Complexity (AVERAGE and WORST):
            * Access: O(1)
            * Search: O(n)
            * Insertion: O(n)
            * Deletion: O(n)
        """
        self._size: int = INITIAL_SIZE
        self._capacity: int = self._set_capacity(capacity)
        self.data: list[T] = []

    def __iter__(self) -> T:
        return iter(self.data)

    def __getitem__(self, index: int) -> T:
        """
        Returns item at given index.

        - Complexity: O(1)

        Args:
            index: Index where the item is found.

        Raises:
            - IndexError: If index out of range.
            - ValueError: If the index is not an Integer.

        Return:
            T: The found item
        """
        self._validate_index_type(index)
        if index == -1:
            return self._get_last_item()

        for i, value in enumerate(self.data):
            if i == index:
                return value
        raise IndexError(INDEX_OUT_OF_RANGE_ERROR)

    def append(self, value: T) -> None:
        """
        Add an item to the end of the list.

        - Complexity: O(1)

        Args:
            value: The value that will be add.
        """
        self._upsize_capacity()
        if self.is_empty():
            self.data = [value]
        else:
            self.data += [value]
        self._size += 1

    def insert(self, index: int, value: T) -> None:
        """
        Insert an item at a given position.

            * [index = 0]: Insert the item at the beginning - Complexity: O(n);\n
            * [index > current size or index < 0]: Insert the item at the end
                (is equivalent to append(value)) - Complexity: O(1);\n
            * [index < current size or any other numbers]: Insert the item among
                other elements - Complexity: O(n).
        --------

        Args:
            index: The index where you want to insert the value.
            value: The value that will be insert.

        Raises:
            ValueError: If the index is not an Integer
        """
        self._validate_index_type(index)

        self._upsize_capacity()
        if index == INITIAL_SIZE:  # Insert the element at the beginning
            shifted_array = [value]
            for item in self.data:
                shifted_array += [item]
            self.data = shifted_array
            self._size += 1
        elif (
            index > self._size or index < INITIAL_SIZE
        ):  # Insert the element at the end
            self.append(value)
        else:  # Insert the element among other elements
            left_content = self.data[:index]
            right_content = self.data[index:]

            left_content += [value]
            for item in right_content:
                left_content += [item]

            self.data = left_content
            self._size += 1

    def pop(self, index: int = None) -> T:
        """
        Remove a value at the end, and return it.

        - Complexity:
            * O(1) if the index is at the end.
            * O(n) from specific index.

        Args:
            index: Index where the item is found.

        Raises:
            - IndexError:
                * If there is nothing in the array;
                * And if index is greater than size.
            - ValueError: If the index is not an integer.

        Return:
            T: The removed value
        """
        if index is not None:
            self._validate_index_type(index)

        if self.is_empty() or index is not None and index > self._size:
            raise IndexError(INDEX_OUT_OF_RANGE_ERROR)

        target_value: T = None
        if index is None or index < INITIAL_SIZE or index == self._size - 1:
            self._downsize_capacity()
            target_value = self.data[-1]
            del self.data[-1]
            self._size -= 1
        else:
            # Searching...
            for value in self.data:
                if value == self.data[index]:
                    target_value = value
                    self._downsize_capacity()
                    self._reorganize(index)

                    del self.data[-1]
                    self._size -= 1
                    break
            else:
                raise IndexError(INDEX_OUT_OF_RANGE_ERROR)

        return target_value

    def remove(self, target_value: T) -> None:
        """
        Remove the first item from the array that is equal to the target value.

        Complexity: O(n)

        Args:
            target_value: value you want to remove.

        Raises:
            ValueError:
                * if no matching value is found.
        """
        index_found = self.find(target_value)
        self._downsize_capacity()
        self._reorganize(index_found)
        del self.data[-1]
        self._size -= 1

    def remove_all(self, target_values: list[T]) -> None:
        """
        Remove values from the array that are equal to the list of target values.

        Complexity: O(n)

        Args:
            target_values: list of values you want to remove.

        Raises:
            ValueError: If no matching value is found.
        """
        self._downsize_capacity()
        for target in target_values:
            index_found = self.find(target)
            self._reorganize(index_found)
            del self.data[-1]
            self._size -= 1

    def find(self, value: T) -> int:
        """
        Search for the given value and returns its index.

        - Complexity: O(n)

        Args:
            value: value you want to find.

        Raises:
            ValueError: If the value is not found.
        """
        for index, item in enumerate(self.data):
            if item == value:
                break
        else:
            raise ValueError(NOT_MATCH_VALUE_FOUND_ERROR)
        return index

    def is_empty(self) -> bool:
        """Check if the array is empty."""
        return self._size == INITIAL_SIZE

    def size(self) -> int:
        return self._size

    def capacity(self) -> int:
        return self._capacity

    def _reorganize(self, target_index: int) -> None:
        for i in range(target_index, self._size - 1):
            self.data[i] = self.data[i + 1]

    def _get_last_item(self):
        return self.data[::-1][0]

    @staticmethod
    def _set_capacity(capacity: int) -> int:
        """
        Arrays will be created with a minimium of 16 bits (2 bytes) and
        a growth factor of 2.

        Examples:
            * capacity = 8 ->  Real capacity: 16 bits (2 bytes);
            * capacity = 9 ->  Real capacity: 32 bits (4 bytes);
            * capacity = 17 -> Real capacity: 64 bits (8 bytes);
            * ....
        """
        if capacity < MIN_INITIAL_CAPACITY:
            raise ValueError(
                f"The minimium initial capacity must be greater than or equal to {MIN_INITIAL_CAPACITY}"
            )

        real_capacity = MIN_REAL_CAPACITY
        while capacity > real_capacity // GROWTH_FACTOR:
            real_capacity *= GROWTH_FACTOR

        return real_capacity

    def _upsize_capacity(self) -> None:
        if self._size == self._capacity:
            old_capacity = self._capacity
            self._capacity = self._set_capacity(old_capacity)

    def _downsize_capacity(self) -> None:
        if self._size < self._capacity // SHRINK_FACTOR:
            old_capacity = self._capacity
            new_capacity = self._capacity // GROWTH_FACTOR

            if new_capacity < MIN_REAL_CAPACITY:
                new_capacity = MIN_REAL_CAPACITY

            if new_capacity != old_capacity:
                self._capacity = new_capacity

    @staticmethod
    def _validate_index_type(index: int) -> None:
        if not isinstance(index, int):
            raise ValueError("Index must be a integer!")


import sys
import numpy as np
import json
import os
from collections import Counter, defaultdict
from difflib import SequenceMatcher
from env import DATA_DIR

def validate_rels(graph):
    entities = graph['entities']
    relations = graph['relations']
    for e1, e2 in relations:
        assert e1 in entities
        assert e2 in entities

def _get_geid2peids(alignment_fname):
    geid2peids = defaultdict(set)
    with open(alignment_fname) as fin:
        for line in fin:
            if line.startswith('\t'):
                peid = line.strip().split('\t')[1][4:-1]
                geid2peids[geid].add(peid)
            elif line.startswith('EID'):
                geid = line.strip().split('\t')[0][4:-1]
            else:
                continue
    return geid2peids

""" get entity cluster type (type with highest frequency) """
def _get_cluster_id2type(cluster_id2pred_eids, pred_eids):
    cluster_id2type = {}
    for cid in cluster_id2pred_eids:
        type2freq = defaultdict(int)
        for eid in cluster_id2pred_eids[cid]:
            type2freq[pred_eids[eid][0]] += pred_eids[eid][2]
        best_type = sorted(type2freq.items(), key=lambda x: x[-1], reverse=True)[0][0]
        cluster_id2type[cid] = best_type
    return cluster_id2type

""" get relation type (all predicted types between each pairs in 2 entity clusters) """
def _get_rel_cluster2types(pred_rels_in_cluster_ids, pred_rels, cluster_id2pred_eids):
    rel_cluster2types = defaultdict(set)
    for r in pred_rels_in_cluster_ids:
        cid1, cid2 = r
        for eid1 in cluster_id2pred_eids[cid1]:
            for eid2 in cluster_id2pred_eids[cid2]:
                if (eid1, eid2) in pred_rels:
                    for rtype in pred_rels[(eid1, eid2)]:
                        rel_cluster2types[r].add(rtype)
    assert len(rel_cluster2types) == len(pred_rels_in_cluster_ids)
    return rel_cluster2types

def eval_graph_hard_align(gold_graph, pred_graph, docid, corpus_type, match_type=False):
    """ getting alignment map """
    assert corpus_type in ['valid', 'test', 'auto_test']
    geid2peids = _get_geid2peids('{}/json/{}/processed/aligned_entities/{}.txt'.format(DATA_DIR, corpus_type, docid))

    # each graph should be ents: {eid: (type, names, freq)} and rels: {(eid1, eid2): [type1, type2, ...]}
    gold_eids = gold_graph['entities']
    pred_eids = pred_graph['entities']
    gold_rels = gold_graph['relations']
    pred_rels = pred_graph['relations']

    """ assign each predicted node to its corresponding cluster """
    cluster_id2pred_eids = defaultdict(set)
    unaligned_cluster_id = 0
    pred_eid2cluster_id  = {}
    # for each predicted entity eid (node), find its alignment to target graph as the cluster (id as the eid in target graph)
    # for any unaligned node, it simply become a single node with cluster id as "unaligned_id"
    for peid in pred_eids:
        aligned = False
        for geid in gold_eids:
            if peid in geid2peids[geid]:
                cluster_id2pred_eids[geid].add(peid)
                pred_eid2cluster_id[peid] = geid
                aligned = True
                break
        if not aligned:
            cluster_id = 'unaligned_{}'.format(unaligned_cluster_id)
            cluster_id2pred_eids[cluster_id].add(peid)
            pred_eid2cluster_id[peid] = cluster_id
            unaligned_cluster_id += 1

    """ assign entity type for each cluster """
    cluster_id2type = _get_cluster_id2type(cluster_id2pred_eids, pred_eids) # TODO

    """ count the number of target nodes being predicted """
    matched_eid_gold = 0
    for geid in gold_eids:
        if geid in cluster_id2type and (not match_type or gold_eids[geid][0] == cluster_id2type[geid]):
            matched_eid_gold += 1

    """ count the number of predicted entity clusters being correct """
    """ count total number of aligned clusters and entities in all clusters : for calculating entity dup rate """
    matched_eid_pred = 0
    ent_dup_rate = (0, 0)
    for cluster_id in cluster_id2pred_eids:
        if cluster_id in gold_eids:
            if not match_type:
                matched_eid_pred += 1
            else:
                if cluster_id2type[cluster_id] == gold_eids[cluster_id][0]:
                    matched_eid_pred += 1
            ent_dup_rate = (ent_dup_rate[0]+len(cluster_id2pred_eids[cluster_id]), ent_dup_rate[1]+1)
        else:
            assert len(cluster_id2pred_eids[cluster_id]) == 1
    # matched_eid_gold should equal matched_eid_pred
    assert matched_eid_pred == matched_eid_gold
    if ent_dup_rate[1] == 0:
        ent_dup_rate = 1.0
    else:
        ent_dup_rate = ent_dup_rate[0]*1.0/ent_dup_rate[1]

    """ represent predicted relations in cluster ids (relations between the same clusters with the same direction are merged) """
    pred_rels_in_cluster_ids = set()
    for peid1, peid2 in pred_rels:
        if not match_type:
            rel_in_cluster_ids = frozenset([pred_eid2cluster_id[peid1], pred_eid2cluster_id[peid2]])
        else:
            rel_in_cluster_ids = (pred_eid2cluster_id[peid1], pred_eid2cluster_id[peid2])
        pred_rels_in_cluster_ids.add(rel_in_cluster_ids)

    """ assign relation type(s) to each relation if match_type == True"""
    if match_type:
        rel_cluster2types = _get_rel_cluster2types(pred_rels_in_cluster_ids, pred_rels, cluster_id2pred_eids)

    """ count number of target relations being predicted """
    matched_rel_gold = 0
    gold_rels_without_type_and_direction = set()
    for geid1, geid2 in gold_rels:
        matched_found = False
        if not match_type:
            key = frozenset([geid1, geid2])
            # do not count multiple times for the same key
            if key in gold_rels_without_type_and_direction:
                continue
            gold_rels_without_type_and_direction.add(key)
            if key in pred_rels_in_cluster_ids:
                matched_rel_gold += 1
        else:
            key = (geid1, geid2)
            if key in rel_cluster2types:
                for gold_type in gold_rels[key]:
                    if gold_type in rel_cluster2types[key]:
                        matched_rel_gold += 1

    """ count number of predicted relations being correct """
    matched_rel_pred = 0
    for pred_rel in pred_rels_in_cluster_ids:
        if not match_type:
            if pred_rel in gold_rels_without_type_and_direction:
                matched_rel_pred += 1
        else:
            if pred_rel in gold_rels:
                for pred_type in rel_cluster2types[pred_rel]:
                    if pred_type in gold_rels[pred_rel]:
                        matched_rel_pred += 1
    assert matched_rel_pred ==matched_rel_gold

    matched_eid_gold = matched_eid_gold*1.0
    matched_eid_pred = matched_eid_pred*1.0
    matched_rel_gold = matched_rel_gold*1.0
    matched_rel_pred = matched_rel_pred*1.0

    prec_ent = matched_eid_pred/(len(cluster_id2pred_eids)+1e-6)
    recall_ent = matched_eid_gold/(len(gold_eids)+1e-6)
    f1_ent = 2*(prec_ent*recall_ent)/(prec_ent+recall_ent+1e-6)

    if not match_type:
        prec_rel = matched_rel_pred/(len(pred_rels_in_cluster_ids)+1e-6)
        recall_rel = matched_rel_gold/(len(gold_rels_without_type_and_direction)+1e-6)
        f1_rel = 2*(prec_rel*recall_rel)/(prec_rel+recall_rel+1e-6)
    else:
        # count each (edge link, relation type) as 1
        total = sum([len(rel_cluster2types[r]) for r in rel_cluster2types])
        prec_rel = matched_rel_pred/(total+1e-6)
        recall_rel = matched_rel_gold/(sum([len(gold_rels[r]) for r in gold_rels])+1e-6)
        f1_rel = 2*(prec_rel*recall_rel)/(prec_rel+recall_rel+1e-6)

    res = {"entity" : {"precision":prec_ent, "recall":recall_ent, "f1":f1_ent}, "relation" : {"precision":prec_rel, "recall":recall_rel, "f1":f1_rel}, "ent_dup_rate":ent_dup_rate}
    return res

def get_entities_map(full_graph, pred_ents):
    eid2mentions = full_graph['entity_cluster_id_to_mentions']
    eid2names = full_graph['entity_cluster_id_to_string_names']
    eid2type = {}
    eid2freq = {}
    for eid in eid2mentions:
        mentions = eid2mentions[eid]
        eid2type[eid] = Counter([m[-1] for m in eid2mentions[eid]]).most_common(1)[0][0]
        eid2freq[eid] = len(eid2mentions[eid])
    results = {}
    for eid in pred_ents:
        if str(eid) not in eid2freq:
            continue
        results[str(eid)] = (eid2type[str(eid)], eid2names[str(eid)], eid2freq[str(eid)])
    return results

def find_valid_relations(full_graph, pred_ents):
    rels = full_graph['relations']
    rel2mentions = full_graph['relation_id_to_mentions']
    valid_eid_set = set(pred_ents.keys())
    valid_rels = defaultdict(set)
    for rid in rels:
        assert len(rels[rid]) == 1
        eid1, eid2, rtype = rels[rid][0].split('_')
        if eid1 in valid_eid_set and eid2 in valid_eid_set and eid1 != eid2:
            valid_rels[(eid1, eid2)].add(rtype)
    return valid_rels

def find_valid_relations_from_predicted_rels(predicted_rels, pred_ents, n_r_types, n_ents):
    assert n_r_types == 7
    rid2type = {0:'COMPARE', 1:'PART-OF', 2:'CONJUNCTION', 3:'EVALUATE-FOR', 4:'FEATURE-OF', 5:'USED-FOR', 6:'HYPONYM-OF'}

    predicted_rels_indices = (predicted_rels.max(1)[1].flatten() < n_r_types).nonzero().flatten().tolist()
    predicted_rels_types = predicted_rels.max(1)[1]
    valid_eid_set = set(pred_ents.keys())
    valid_rels = defaultdict(set)
    for eid1 in valid_eid_set:
        for eid2 in valid_eid_set:
            idx = int(eid1)*n_ents+int(eid2)
            if idx in predicted_rels_indices:
                rid = predicted_rels_types[idx]
                assert rid < n_r_types
                valid_rels[(eid1, eid2)].add(rid2type[int(rid)])

    return valid_rels

def get_docid2full(full_graph_fname):
    docid2full = {}
    with open(full_graph_fname) as f_full:
        for line in f_full:
            full_graph = json.loads(line.strip())
            docid = full_graph['doc_key']
            docid2full[docid] = full_graph
    return docid2full

def get_docid2graph(fname):
    docid2graph = {}
    with open(fname) as fin:
        all_docs = json.load(fin)
        for content in all_docs:
            graph = {'entities':{}, 'relations':{}}
            for eid in content['entities']:
                graph['entities'][eid] = content['entities'][eid]
            for rel in content['relations']:
                eid1, eid2 = rel.split('_')
                graph['relations'][(eid1, eid2)] = content['relations'][rel]
            docid2graph[content['doc_key']] = graph
    return docid2graph

# each graph should be ents: {eid: (type, names, freq)} and rels: {(eid1, eid2): [type1, type2, ...]}
def eval_graph(gold_graph, pred_graph, docid, corpus_type, match_type=False):
    validate_rels(gold_graph)
    validate_rels(pred_graph)
    edit_hard_results = eval_graph_hard_align(gold_graph, pred_graph, docid, corpus_type, match_type=match_type)

    return edit_hard_results

def eval(gold_fname, pred_fname, corpus_type, match_type):
    docid2gold = get_docid2graph(gold_fname)
    docid2pred = get_docid2graph(pred_fname)
    ave_edit_hard = {"entity" : {"precision":0.0, "recall":0.0, "f1":0.0}, "relation" : {"precision":0.0, "recall":0.0, "f1":0.0}, "ent_dup_rate":0.0}

    n_docs = len(docid2pred)
    for docid in docid2pred:
        edit_hard_results = eval_graph(docid2gold[docid], docid2pred[docid], docid, corpus_type, match_type=match_type)
        ave_edit_hard['entity']['precision'] += edit_hard_results['entity']['precision']
        ave_edit_hard['entity']['recall'] += edit_hard_results['entity']['recall']
        ave_edit_hard['entity']['f1'] += edit_hard_results['entity']['f1']
        ave_edit_hard['relation']['precision'] += edit_hard_results['relation']['precision']
        ave_edit_hard['relation']['recall'] += edit_hard_results['relation']['recall']
        ave_edit_hard['relation']['f1'] += edit_hard_results['relation']['f1']
        ave_edit_hard['ent_dup_rate'] += edit_hard_results['ent_dup_rate']
        new_edit_hard_results = dict(edit_hard_results)
        for key in edit_hard_results:
            if key == 'ent_dup_rate':
                new_edit_hard_results[key] = f"{edit_hard_results[key]:.3f}"
            else:
                for key2 in edit_hard_results[key]:
                    new_edit_hard_results[key][key2] = f"{edit_hard_results[key][key2]:.3f}"

    ave_edit_hard['entity']['precision'] /= n_docs
    ave_edit_hard['entity']['precision'] = round(ave_edit_hard['entity']['precision'], 4)

    ave_edit_hard['entity']['recall'] /= n_docs
    ave_edit_hard['entity']['recall'] = round(ave_edit_hard['entity']['recall'], 4)

    ave_edit_hard['entity']['f1'] /= n_docs
    ave_edit_hard['entity']['f1'] = round(ave_edit_hard['entity']['f1'], 4)

    ave_edit_hard['relation']['precision'] /= n_docs
    ave_edit_hard['relation']['precision'] = round(ave_edit_hard['relation']['precision'], 4)

    ave_edit_hard['relation']['recall'] /= n_docs
    ave_edit_hard['relation']['recall'] = round(ave_edit_hard['relation']['recall'], 4)

    ave_edit_hard['relation']['f1'] /= n_docs
    ave_edit_hard['relation']['f1'] = round(ave_edit_hard['relation']['f1'], 4)

    ave_edit_hard['ent_dup_rate'] /= n_docs
    ave_edit_hard['ent_dup_rate'] = round(ave_edit_hard['ent_dup_rate'], 4)

    print('========== EVALUATION RESULTS : ===========')
    print(ave_edit_hard)

if __name__ == "__main__":
    gold_graph_fname = sys.argv[1]
    pred_graph_fname = sys.argv[2]
    corpus_type = sys.argv[3]
    assert corpus_type in ['valid', 'test', 'auto_test']
    typed_eval = sys.argv[4]=='1'

    eval(gold_graph_fname, pred_graph_fname, corpus_type, typed_eval)


import os
import math

import pygame as pg

from resources import Spritesheet

class Golfer(pg.sprite.Sprite):

    def __init__(self, screen, clock, club):
        """Initialize the golfer.

        :param screen: The pygame display
        :param clock: The pygame game clock
        :param club: The golfer's club
        """
        super(Golfer, self).__init__()
        self.screen = screen
        self.clock = clock
        self.size = 110, 200
        # Set the radius for detecting collisions
        self.radius = 50

        # Set the sprite images
        self.images, self.image_functions = self.get_images()
        self.image_index = 0
        self.set_image()

        # Set the rectangle, or the box where it is drawn
        self.rect = self.surf.get_rect()
        self.rect.bottom = self.screen.get_size()[1] - 60
        self.rect.right = self.screen.get_size()[0] / 2 - 50
        self.vector = [0, 0]
        self.speed = 0
        self.club = club
        # Set the animation stuff
        self.is_in_swing = False
        self.animation_frames = 5  # the frames per animation
        self.current_frame = 0

    def get_images(self):
        """Get all images defined by the spritesheet and return them.

        :return: A list pygame.Surface and index dict
        :rtype: list of pygame.Surface and dict of int: int
        """
        ss = Spritesheet(
            os.path.join(os.getcwd(), 'resources', 'golfer_sheet.png')
        )
        strip = ss.load_strip((0,0,220,400), 4)
        scaled_strip = []
        for sprite in strip:
            scaled_strip.append(
                pg.transform.scale(sprite, self.size)
            )
        return scaled_strip, {
                0: self.set_resting,
                1: self.set_mid_swing,
                2: self.set_full_swing,
                3: self.set_mid_swing,
                4: self.set_resting,
                5: self.set_followthru
            }

    def set_image(self, image_index=0):
        if image_index is not None and \
           image_index in range(0, len(self.images)):
            self.surf = self.images[image_index]

    def set_resting(self):
        self.set_image(image_index=0)
        self.club.set_resting()

    def set_mid_swing(self):
        self.set_image(image_index=1)
        self.club.set_mid_swing()

    def set_full_swing(self):
        self.set_image(image_index=2)
        self.club.set_full_swing()

    def set_followthru(self):
        self.set_image(image_index=3)
        self.club.set_followthru()

    def update_frame_dependent(self):
        """Update the sprite image every 5 frames during a swing."""
        power, speed = 0, 0
        self.current_frame += 1
        if self.current_frame >= self.animation_frames:
            self.current_frame = 0
            if self.image_index == len(self.image_functions) - 2:
                power, speed = self.club.power, 0
            if self.image_index == len(self.image_functions) - 1:
                self.is_in_swing = False
            self.image_index = (self.image_index + 1) % \
                               len(self.image_functions)
            self.image_functions[self.image_index]()
        self.club.update()
        self.display()
        return power, speed

    def update(self, pressed_keys, start_swing=False):
        """Update the golfer's location.

        :param pressed_keys: A dict of pressed keys this fram
        """
        if self.is_in_swing:
            return self.update_frame_dependent()
        if start_swing:
            self.is_in_swing = True
            self.update_frame_dependent()
            return None, None
        if pressed_keys[pg.K_UP]:
            self.set_full_swing()
            self.club.set_full_swing()
        if pressed_keys[pg.K_DOWN]:
            self.set_resting()
            self.club.set_resting()
        if pressed_keys[pg.K_LEFT]:
            self.set_followthru()
            self.club.set_followthru()
        if pressed_keys[pg.K_RIGHT]:
            self.set_mid_swing()
            self.club.set_mid_swing()
        self.club.update()
        self.display()
        return None, None

    def calcnewpos(self, rect, vector):
        (angle,z) = vector
        (dx,dy) = (z*math.cos(angle), z*math.sin(angle))
        return rect.move(dx,dy)

    @property
    def image(self):
        return self.surf

    def display(self):
        self.screen.blit(self.surf, self.rect)


import os
from colorama import init, Fore, Back, Style
from .localizations import LOCALS

init(autoreset=True)

class Method:
    """Wrapper for functions which used in app."""

    def __init__(self, descr, function, inline=False):
        self.descr = descr
        self.function = function
        self.inline = inline

class ConsoleApp:
    def __init__(
        self,
        title: str = "ConsoleApp",
        subTitle: str = None,
        printHead: bool = True,
        printMethods: bool = False,
        width: int = 45,
        maxMethodLen: int = 9,
        onStart = None,
        onExit = None,
        lang = "eng",
        **kwargs
    ):
        self.__title = title
        self.__subTitle = subTitle
        self.__printHead = printHead
        self.__printMethods = printMethods
        if width < 40:
            self.__width = 40
        else:
            self.__width = width
        if maxMethodLen < 5:
            self.__maxMethodLen = 6
        else:
            self.__maxMethodLen = maxMethodLen + 1
        self.__onStart = onStart
        self.__onExit = onExit
        if LOCALS.get(lang) is None:
            self.__lang = "eng"
        else:
            self.__lang = lang
        self.__methods = kwargs

    def changeSubTitle(self, newSub: str):
        self.__subTitle = newSub
        self.__clear()

    def __printLine(self):
        print("─" * self.__width)

    def __helpMethod(self):
        for title, method in self.__methods.items():
            print("{0:>{2}} - {1}".format(title, method.descr, self.__maxMethodLen))
        print("{0:>{2}} - {1}".format("help", LOCALS[self.__lang]["help"], self.__maxMethodLen))
        print("{0:>{2}} - {1}".format("clear", LOCALS[self.__lang]["clear"], self.__maxMethodLen))
        print("{0:>{2}} - {1}".format("exit", LOCALS[self.__lang]["exit"], self.__maxMethodLen))

    def __printMethodsFunc(self):
        if not self.__printHead:
            self.__printLine()

        methods = list(self.__methods.keys())
        methods.extend(["help", "clear", "exit"])
        for index, method in enumerate(methods):
            if not (index + 1) % (self.__width // self.__maxMethodLen) or index == len(methods) - 1:
                print(f"{method:<{self.__maxMethodLen}}")
            else:
                print(f"{method:<{self.__maxMethodLen}}", end="")

        self.__printLine()

    def __printHeadFunc(self):
        print(Fore.CYAN + f"{self.__title:^{self.__width}}")
        if self.__subTitle:
            print(Fore.GREEN + f"{self.__subTitle:^{self.__width}}")

        hint = LOCALS[self.__lang]["hint"]
        print(f"{hint:^{self.__width}}")
        self.__printLine()

    def __clear(self):
        os.system("cls")
        if self.__printHead:
            self.__printHeadFunc()
        if self.__printMethods:
            self.__printMethodsFunc()

    def run(self):
        os.system(f"mode con cols={self.__width} lines={self.__width//2}")

        if self.__printHead:
            self.__printHeadFunc()
        if self.__printMethods:
            self.__printMethodsFunc()
        if self.__onStart:
            try:
                self.__onStart()
            except Exception as exc:
                print(Fore.RED + LOCALS[self.__lang]["error"], exc)

        while True:
            userInput = str(input(">"))
            userInput = userInput.split()

            try:
                command = userInput[0]
            except IndexError as exc:
                command = ""

            method = self.__methods.get(command)
            if method is not None:
                self.__printLine()
                if not method.inline:
                    try:
                        method.function()
                    except Exception as exc:
                        print(Fore.RED + LOCALS[self.__lang]["error"], exc)
                else:
                    try:
                        method.function(userInput[1:])
                    except IndexError as exc:
                        print(Fore.YELLOW + LOCALS[self.__lang]["args_error"])
                    except Exception as exc:
                        print(Fore.RED + LOCALS[self.__lang]["error"], exc)
                self.__printLine()
            elif command == "exit":
                if self.__onExit:
                    try:
                        self.__onExit()
                    except Exception as exc:
                        print(Fore.RED + LOCALS[self.__lang]["error"], exc)
                break
            elif command == "help":
                self.__printLine()
                self.__helpMethod()
                self.__printLine()
            elif command == "clear":
                self.__clear()
            else:
                print(LOCALS[self.__lang]["unknown_command"])


#!/usr/bin/env python
"""
File: get_circ_exp.py
Description: Standardize circRNAs expression with CPM.
CreateDate: 2023/3/4
Author: xuwenlin
E-mail: wenlinxu.njfu@outlook.com
"""
import click
from pybioinformatic import read_in_gene_expression_as_dataframe, Displayer
displayer = Displayer(__file__.split('/')[-1], version='1.0.0')

def main(BSJ_matrix_file, out_file):
    df = read_in_gene_expression_as_dataframe(BSJ_matrix_file)
    if isinstance(df, str):
        click.echo(df, err=True)
        exit()
    read_sum = df.sum(axis=0)
    df = df.div(read_sum, axis=1) * 10 ** 6
    df.to_csv(f'./{out_file}.csv')

@click.command(context_settings=dict(help_option_names=['-h', '--help']))
@click.option('-i', '--bsj_matrix', 'bsj_matrix',
              metavar='<exp file>', required=True,
              help='Input BSJ matrix file. (support format: txt, xls, xlsx, csv)')
@click.option('-o', '--output_prefix', 'output_prefix',
              metavar='<str>', default='circ_CPM', show_default=True,
              help='Output file prefix.')
@click.option('-V', '--version', 'version', help='Show author and version information.',
              is_flag=True, is_eager=True, expose_value=False, callback=displayer.version_info)
def run(bsj_matrix, output_prefix):
    """Standardize circRNAs expression with CPM."""
    main(bsj_matrix, output_prefix)

if __name__ == '__main__':
    run()


#!/usr/bin/env python
# coding=utf-8
'''
@author: yandeqing
@date:  2020/7/29 16:58
'''
import time

import requests

import Config
import industry_moneyflow_eastmoney

def upload_one(dest_info, type):
    dest_info['type'] = type
    response = requests.post(Config.url, json=dest_info, headers={'Connection': 'close'})
    print(f"insert {dest_info}{response.text}")

def is_trade_day(debug=False):
    if debug:
        return True
    date = industry_moneyflow_eastmoney.get_latest_date()
    time_strftime = time.strftime("%Y-%m-%d", time.localtime())
    if date != time_strftime:
        print(f"今天{time_strftime}不是交易日")
    return date == time_strftime

if __name__ == '__main__':
    print(is_trade_day())