import argparse,os,time,sys
import numpy as np
import tensorflow as tf
from chiron_input import read_data_for_eval
from utils.easy_assembler import simple_assembly
from utils.easy_assembler import simple_assembly_qs
#from utils.easy_assembler import section_decoding
from cnn import getcnnfeature
from cnn import getcnnlogit
from rnn import rnn_layers
from utils.unix_time import unix_time

import tvm
from tvm import relay
import tvm.relay.testing.tf as tf_testing

def inference(x,seq_length,training):
    cnn_feature = getcnnfeature(x,training = training)
    feashape = cnn_feature.get_shape().as_list()
    ratio = FLAGS.segment_len/feashape[1]
    #logits = rnn_layers(cnn_feature,seq_length/ratio,training,class_n = 5 )
#   logits = rnn_layers_one_direction(cnn_feature,seq_length/ratio,training,class_n = 4**FLAGS.k_mer+1 )
    logits = getcnnlogit(cnn_feature)
    return logits,ratio

def path_prob(logits):
    top2_logits = tf.nn.top_k(logits,k=2)[0]
    logits_diff = tf.slice(top2_logits,[0,0,0],[FLAGS.batch_size,FLAGS.segment_len,1])-tf.slice(top2_logits,[0,0,1],[FLAGS.batch_size,FLAGS.segment_len,1])
    prob_logits = tf.reduce_mean(logits_diff,axis = -2)
    return prob_logits

def load_graph(frozen_graph_filename):
    with tf.gfile.GFile(frozen_graph_filename, "rb") as f:
        graph_def = tf.GraphDef()
        graph_def.ParseFromString(f.read())

    with tf.Graph().as_default() as graph:
        tf.import_graph_def(graph_def,name="prefix")
    return graph

def evaluation():
    x = tf.placeholder(tf.float32,shape = [FLAGS.batch_size,FLAGS.segment_len])
    seq_length = tf.placeholder(tf.int32, shape = [FLAGS.batch_size])
    training = tf.placeholder(tf.bool)

    logits,_ = inference(x,seq_length,training = training)
    predict = tf.nn.ctc_greedy_decoder(tf.transpose(logits,perm=[1,0,2]),seq_length,merge_repeated = True)
    prob = path_prob(logits)

    target = 'llvm'
    target_host = 'llvm'
    layout = None
    ctx = tvm.cpu(0)

    #graph = load_graph("frozen_model.pb")
    graph = load_graph("optimized_model.pb")
    x = graph.get_tensor_by_name('prefix/Placeholder_1:0')
    y = graph.get_tensor_by_name('prefix/cnnlogits_rs:0')

    with tf.Session(graph=graph) as sess:

     file_list = os.listdir(FLAGS.input)
     file_dir = FLAGS.input

     shape_dict = None
     sym, params = relay.frontend.from_tensorflow(sess.graph_def, layout=layout, shape=shape_dict)
     print ("Tensorflow protobuf imported to relay frontend.")

     for name in file_list:
         start_time = time.time()
         if not name.endswith('.signal'):
             continue
         file_pre = os.path.splitext(name)[0]
         input_path = os.path.join(file_dir,name)
         eval_data = read_data_for_eval(input_path,FLAGS.start,seg_length = FLAGS.segment_len,step = FLAGS.jump)
         reads_n = eval_data.reads_n
         reading_time=time.time()-start_time
         reads = list()
         for i in range(0,reads_n,FLAGS.batch_size):
             batch_x,seq_len,_ = eval_data.next_batch(FLAGS.batch_size,shuffle = False)
             batch_x=np.pad(batch_x,((0,FLAGS.batch_size-len(batch_x)),(0,0)),mode='constant')
             seq_len=np.pad(seq_len,((0,FLAGS.batch_size-len(seq_len))),mode='constant')
             feed_dict = {x:batch_x}
             print(logits_val)

def run(args):
    global FLAGS
    FLAGS=args
    time_dict=unix_time(evaluation)
    print(FLAGS.output)
    print('Real time:%5.3f Systime:%5.3f Usertime:%5.3f'%(time_dict['real'],time_dict['sys'],time_dict['user']))
    meta_folder = os.path.join(FLAGS.output,'meta')
    if os.path.isdir(FLAGS.input):
        file_pre='all'
    else:
        file_pre = os.path.splitext(os.path.basename(FLAGS.input))[0]
    path_meta=os.path.join(meta_folder,file_pre+'.meta')
if __name__=="__main__":
    parser=argparse.ArgumentParser(prog='chiron',description='A deep neural network basecaller.')
    parser.add_argument('-i','--input',default='example_data/output/raw', help="File path or Folder path to the fast5 file.")
    parser.add_argument('-o','--output',default='example_data/output', help = "Output Folder name")
    parser.add_argument('-m','--model', default = 'model/DNA_default',help = "model folder")
    parser.add_argument('-s','--start',type=int,default = 0,help = "Start index of the signal file.")
    parser.add_argument('-b','--batch_size',type = int,default = 1100,help="Batch size for rune processing speed and give a slightly better accuracy but require larger RAM load")
    parser.add_argument('-l','--segment_len',type = int,default = 300, help="Segment length to be divided into.")
    parser.add_argument('-j','--jump',type = int,default = 30,help = "Step size for segment")
    parser.add_argument('-t','--threads',type = int,default = 0,help = "Threads number")
    parser.add_argument('-e','--extension',default = 'fastq',help = "Output file extension.")
    parser.add_argument('--beam',type = int,default = 0, help = "beam width give better decoding result but require longer decoding time.")
    args=parser.parse_args(sys.argv[1:])
    run(args)


import zipfile
from zipfile import ZipFile
import os
from conftest import RES_PATH, TMP_PATH

def test_create_and_read_zip(file_manager):
    file_path = os.listdir(RES_PATH)
    file_name = 'new_arch.zip'

    with zipfile.ZipFile(os.path.join(TMP_PATH, file_name), mode='w', \
                         compression=zipfile.ZIP_DEFLATED) as zip:
        for file in file_path:
            file_add = os.path.join(RES_PATH, file)
            zip.write(file_add)

    with ZipFile(os.path.join(TMP_PATH, 'new_arch.zip')) as zip:
        file_names = [os.path.basename(file) for file in zip.namelist()]
        assert file_names == os.listdir(RES_PATH)


import cv2 as cv
import numpy as np
from scipy import signal
from PIL import Image

def lineSegmentAStar(image):
    #showBinaryImage(image)
    #sys.exit()
    shape = np.shape(image)
    width = shape[1]
    proj = np.sum(image, 1)
    invproj = np.max(proj) - proj
    peaks = signal.find_peaks(invproj, prominence = 0.2*np.max(invproj)) #0.2 is TUNEABLE
    past_peak = 0
    # Initializing AStar
    # Finding the valleys (inverted peaks) that represent lines and run a* algorithm
    j = 0           # counter for lines
    points = []     # list to store edges of lines (paths)

    # compute average line height because we need to scale the parameters:
    line_counter = 1 # initialized to 1 as we skip the first peak in computation
    all_peaks = 0 # stores the cumulative height of all lines
    for peak in peaks[0]:
        if(peak-past_peak>(0.02*shape[0])):
            if past_peak != 0:
                line_counter += 1
                all_peaks += peak-past_peak
            past_peak = peak
    line_average = all_peaks / line_counter
    print("Line average:", line_average)

    past_peak = 0
    for peak in peaks[0]:
        if(peak-past_peak>(0.02*shape[0])):
            print("Running a* for peak ",peak)
            points.insert(j, astar(np.transpose(image), (0, peak), (width-1, peak), line_average))
            #points.insert(j, astar(np.transpose(image), (0, peak), (width, peak)))

        past_peak = peak
        j += 1
    images = []
    imageT = np.transpose(image)
    previous = 0 # set 0 so the first
    # for all lines
    print(len(points))
    for i in range(0,len(points)+1):
        print(i)
        newImage = np.zeros((5000, 5000)) #cropped later
        if i == len(points):
            shape = np.shape(imageT)
            for point in points[i-1]:
                newImage[point[0]][point[1]:shape[1]] = imageT[point[0]][point[1]:shape[1]]
            returnImage = newImage
        else:
            for point in points[i]:
                newImage[point[0]][0:point[1]] = imageT[point[0]][0:point[1]] #abracadabra
            returnImage = newImage - previous
        print(np.sum(returnImage))
        if(np.sum(returnImage)) < 0.01*np.sum(image): #if a line has less than 1% of all black pixels TUNEABLE
            print("continuing because line at peak ",i," is too short")
            continue
        previous = newImage
        #images.append(returnImage)
        coords = cv.findNonZero(returnImage)            # Find all non-zero points (text)
        x, y, w, h = cv.boundingRect(coords)            # Find minimum spanning bounding box
        images.append(returnImage[y:y + h, x:x + w])    # Crop the image and append to all images
    return images

# a star path planning algorithm (borrowed from stack overflow)
from heapq import *

def heuristic(a, b):
    return (b[0] - a[0]) ** 2 + (b[1] - a[1]) ** 2

def astar(array, start, goal, line_height):
    neighbors = [(0, 1), (0, -1), (1, 0), (-1, 0), (1, 1), (1, -1), (-1, 1), (-1, -1)]

    close_set = set()
    came_from = {}
    gscore = {start: 0}
    fscore = {start: heuristic(start, goal)}
    oheap = []

    heappush(oheap, (fscore[start], start))

    while oheap:

        current = heappop(oheap)[1]

        if current == goal:
            data = []
            while current in came_from:
                data.append(current)
                current = came_from[current]
            return data

        close_set.add(current)
        for i, j in neighbors:
            neighbor = current[0] + i, current[1] + j
            tentative_g_score = gscore[current] + heuristic(current, neighbor)
            y_distance = goal[1] - neighbor[1] #This measures diversion from straight line
            if y_distance < 0:
                tentative_g_score += abs(y_distance) * 7.5*line_height # tuneable parameter, penalizes going under straight line
                #print("Going under..")
            if y_distance > 0:
                tentative_g_score += abs(y_distance) * 0 # tuneable parameter, penalizes going over straight line
                #print("going over...")
            if 0 <= neighbor[0] < array.shape[0]:
                if 0 <= neighbor[1] < array.shape[1]:
                    if array[neighbor[0]][neighbor[1]] == 1:
                        tentative_g_score += 150*line_height # negative penalty for black pixels. TUNEABLE parameter!
                else:
                    # array bound y walls
                    continue
            else:
                # array bound x walls
                continue

            if neighbor in close_set and tentative_g_score >= gscore.get(neighbor, 0):
                continue

            if tentative_g_score < gscore.get(neighbor, 0) or neighbor not in [i[1] for i in oheap]:
                came_from[neighbor] = current
                gscore[neighbor] = tentative_g_score
                fscore[neighbor] = tentative_g_score + heuristic(neighbor, goal)
                heappush(oheap, (fscore[neighbor], neighbor))
    print("no route found")
    return []

def showBinaryImage(image):
    image = convertToRGBImage(image)
    image = Image.fromarray(image, 'RGB')
    image.show()

def convertToRGBImage(image):
    (height, width) = np.shape(image)

    data = np.zeros((height, width, 3), dtype=np.uint8)

    for y in range(height):
        for x in range(width):
            if image[y,x] == 0:
                data[y,x] = [255,255,255]
            else:
                data[y,x] = [0,0,0]

    return data


# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.15.2
#   kernelspec:
#     display_name: Python 3 (ipykernel)
#     language: python
#     name: python3
# ---

# # Visualize Cytomining Ecosystem Target Software Data
#
# Visualizations related to Cytomining ecosystem software targeted project data.
#

# +
import asyncio
import itertools
import os
import pathlib
import random
import re
import shutil
from datetime import datetime

import duckdb
import nest_asyncio
import networkx as nx
import netwulf as nw
import numpy as np
import pandas as pd
import plotly
import plotly.colors as pc
import plotly.express as px
import plotly.graph_objects as go
import plotly.io as pio
import pytz
import pyvis
from box import Box
from IPython.display import IFrame
from itables import to_html_datatable
from pandas.api.types import CategoricalDtype
from plotly.offline import plot
from plotly.subplots import make_subplots
from pyppeteer import launch
from pyvis.network import Network
from ydata_profiling import ProfileReport

# set plotly default theme
pio.templates.default = "simple_white"

# get the current datetime
tz = pytz.timezone("UTC")
current_datetime = datetime.now(tz)

# set common str's
title_prefix = "Cytomining Ecosystem Target Software Analysis"

# export locations relative to this notebook
export_dir = "../../docs/cytomining-ecosystem"

# set a color sequence for general use
category_color_sequence = pc.qualitative.Dark2

# set a color sequence for project-specific colors
project_color_sequence = [
    pc.qualitative.Dark2[3],
    pc.qualitative.Dark2[0],
    pc.qualitative.Dark2[1],
]

# set section descriptions
section_descriptions = {
    "Scholarly Papers": """
    This section helps visualize data related to how many
    scholarly papers mention or cite the target software.
    """,
    "Package Usage": """
    This section helps depict usage of the target software
    by leveraging package data from PyPI and Conda.
    """,
    "Software dependents": """
    This section helps show community dependents of the
    targeted software.
    """,
    "Project Visitors": """
    This section helps show how many visitors have
    have interacted with the project over time.
    """,
}
# -

# read data from various files
with duckdb.connect() as ddb:
    tgt_software_df = ddb.query(
        f"""
    SELECT
        pkgstats."Project Name",
        pubstats."Date Created Year",
        pubstats."google_scholar_count",
        pubstats."biorxiv_count",
        pubstats."total_pub_count",
        pubstats."total_pub_count_non_record_linked",
        pkgstats.pypi_downloads_total_unnested AS pypi_downloads_total,
        pkgstats.pypi_downloads_by_month,
        pkgstats.pypi_downloads_monthly_average,
        pkgstats.pypi_downloads_monthly_median,
        pkgstats.conda_downloads_total,
        pkgstats.conda_downloads_by_month,
        pkgstats.conda_downloads_monthly_average,
        pkgstats.conda_downloads_monthly_median,
        ghstats."GitHub Stars",
        ghstats."GitHub Contributor Total Count",
        ghstats."GitHub Network Count",
        ghstats."GitHub Code Search Used By",
        ghstats."GitHub Dependents",
        ghstats."GitHub Dependency Graph Dependents Count",
        ghstats."GitHub Code Search Dependents Count",
        ghstats."GitHub Total Dependents Count",
        ghstats."GitHub Stargazers Count by Month",
        ghstats."GitHub Stargazers Count by Month Average",
        ghstats."GitHub Stargazers Count by Month Median",
        ghstats."GitHub Stargazers Count by Year Average",
        ghstats."GitHub Stargazers Count by Year Median"
    FROM read_parquet('data/loi-target-project-package-metrics.parquet') as pkgstats
    JOIN read_parquet('data/loi-target-project-github-metrics.parquet') as ghstats ON
        pkgstats."Project Name" = ghstats."Project Name"
    JOIN read_parquet('data/loi-target-project-publication-metrics.parquet') as pubstats ON
        pkgstats."Project Name" = pubstats."Project Name"
    """,
    ).df()
tgt_software_df.info()

tgt_software_df.head()

# create list to collect the figures for later display together
fig_collection = []

# +
# Publications for target projects

fig_publications = px.pie(
    title="Pycytominer Publication Mentions or Citations",
    data_frame=tgt_software_df[["google_scholar_count", "biorxiv_count"]].T,
    values=0,
    names=["Google Scholar Results", "BioRxiv Preprints"],
    hover_name=["Google Scholar Results", "BioRxiv Preprints"],
    width=500,
    height=500,
)
fig_publications.update_traces(textinfo="value")

fig_collection.append(
    {
        "plot": fig_publications,
        "description": """
        This plot shows how many publications mention or cite Pycytominer
        as a reference. We used Google Scholar and BioRxiv as search resources
        for this work. Google Scholar is a scholarly article aggregator and search service.
        BioRxiv is a pre-print server with biological focus.
        """,
        "findings": """
        We observe that Pycytominer appears in 22 Google Scholar results and 14 BioRxiv
        preprints. This demonstrates how Pycytominer is already being used in both
        previous and ongoing research efforts.
        """,
        "section": "Scholarly Papers",
    }
)

fig_publications.show()

# +
# visualize package data

# Create a horizontal bar chart using Plotly Express
fig_packages = px.bar(
    data_frame=tgt_software_df.sort_values("Project Name"),
    x=["pypi_downloads_total", "conda_downloads_total"],
    y="Project Name",
    orientation="h",
    width=1200,
    height=500,
    color_discrete_sequence=[category_color_sequence[2], category_color_sequence[0]],
)

fig_packages.update_layout(
    title="PyPI and Conda Package Downloads by Project",
    xaxis_title="Package Downloads",
    showlegend=False,
)

fig_collection.append(
    {
        "plot": fig_packages,
        "description": """This plot shows the total number of PyPI and Conda downloads by project.
        PyPI (Python Package Index) is a platform which is commonly used to download Python packages.
        Conda is an environment and package management service which is commonly used for data science
        projects.
        """,
        "findings": """This plot demonstrates how pycytominer has many downloads through both PyPI and Conda
        (with slightly more Conda downloads). CytoTable and CytoSnake are shown with strong early download counts
        at a much lower level (perhaps in alignment with their project age, which is less than pycytominer).
        """,
        "section": "Package Usage",
    }
)

# Show the plot
fig_packages.show()

# +
pypi_monthly_data = pd.concat(
    pd.DataFrame(
        [
            dict(month_entry, **{"Project Name": project_name})
            for month_entry in monthly_data
        ]
    )
    for project_name, monthly_data in zip(
        tgt_software_df["Project Name"], tgt_software_df["pypi_downloads_by_month"]
    )
)
fig_pypi_monthly = px.line(
    data_frame=pypi_monthly_data,
    title=f"Project PyPI Package Downloads by Month",
    x="download_month",
    y="download_count",
    color="Project Name",
    width=1200,
    height=500,
    markers=True,
    symbol_sequence=["square"],
    color_discrete_sequence=[project_color_sequence[0], project_color_sequence[2]],
)
fig_pypi_monthly.update_layout(
    xaxis_title="Month",
    yaxis_title="Downloads",
)

fig_collection.append(
    {
        "plot": fig_pypi_monthly,
        "description": """This plot shows how many PyPI package downloads per month by project.
        CytoSnake is excluded from this plot as it is not available on PyPI.
        """,
        "findings": """Pycytominer has a consistent and trending upward number of monthly PyPI package downloads.
        CytoTable shows activity in this area, possibly trending upwards.
        """,
        "section": "Package Usage",
    }
)

fig_pypi_monthly.show()

# +
conda_monthly_data = pd.concat(
    pd.DataFrame(
        [
            dict(month_entry, **{"Project Name": project_name})
            for month_entry in [
                {"download_month": key, "download_count": val}
                for key, val in monthly_data.items()
            ]
        ]
    )
    for project_name, monthly_data in zip(
        tgt_software_df["Project Name"], tgt_software_df["conda_downloads_by_month"]
    )
    if isinstance(monthly_data, dict)
)
fig_conda_monthly = px.line(
    data_frame=conda_monthly_data,
    title=f"Project Conda Package Downloads by Month",
    x="download_month",
    y="download_count",
    color="Project Name",
    width=1200,
    height=500,
    markers=True,
    symbol_sequence=["diamond"],
    color_discrete_sequence=[project_color_sequence[0], project_color_sequence[1]],
)
fig_conda_monthly.update_layout(
    xaxis_title="Month",
    yaxis_title="Downloads",
)

fig_collection.append(
    {
        "plot": fig_conda_monthly,
        "description": """This plot shows how many Conda package downloads per month by project.
        CytoTable is excluded from this plot as it is not available on Conda.
        """,
        "findings": """We can observe from this plot that pycytominer is actively downloaded on monthly basis.
        There is a large spike in download count for pycytominer in 2022 with lower counts in 2023.
        CytoSnake shows strong signs of good adoption for a relatively young project (in comparison with pycytominer).
        """,
        "section": "Package Usage",
    }
)

fig_conda_monthly.show()

# +
# visualize dependents data

# Create a horizontal bar chart using Plotly Express
fig_dependents_count = px.bar(
    data_frame=tgt_software_df[
        tgt_software_df["GitHub Total Dependents Count"] != 0
    ].sort_values("Project Name"),
    x="GitHub Total Dependents Count",
    y="Project Name",
    orientation="h",
    width=1200,
    height=500,
    color_discrete_sequence=[pc.qualitative.Vivid[2]],
)

fig_dependents_count.update_layout(
    title="GitHub Dependency Graph and Code Search Dependents Count by Project",
    xaxis_title="Dependents",
    legend_title_text="Dependent Type",
)

fig_collection.append(
    {
        "plot": fig_dependents_count,
        "description": """This plot shows dependents counts of the projects by leveraging the Dependency Graph and
        code search features of GitHub. Overlaps by organization and project name are removed to help count unique
        dependents.
        """,
        "findings": """We observe that Pycytominer has many existing dependents on GitHub.
        CytoTable has a smaller number of dependents which may be relative to the project age.
        """,
        "section": "Software dependents",
    }
)

# Show the plot
fig_dependents_count.show()

# +
# create a dependent graph to visualize project connections
dependent_data = pd.concat(
    pd.DataFrame(
        [
            {
                "Project Name": project_name,
                "dependent": dependent.replace("cytomining/CytoTable", "CytoTable"),
                "color": "#E7358A"
                if project_name == "pycytominer"
                else "#D9651D"
                if project_name == "CytoTable"
                else "#698DC7",
            }
            for dependent in list(
                set(
                    code_search
                    + [
                        repo["name"]
                        for repo in graph_results["all_public_dependent_repos"]
                    ]
                )
            )
        ]
    )
    for project_name, code_search, graph_results in zip(
        tgt_software_df["Project Name"],
        tgt_software_df["GitHub Code Search Used By"],
        tgt_software_df["GitHub Dependents"],
    )
)

dependent_data

nx_dependent_data = nx.from_pandas_edgelist(
    df=dependent_data, source="dependent", target="Project Name", edge_attr=["color"]
)

nt_dependents = Network(height="500px", width="1200px")
# populates the nodes and edges data structures
nt_dependents.from_nx(nx_dependent_data)
nt_dependents.nodes = [
    dict(node, **{"color": "#E7358A"})
    if node["id"] == "pycytominer"
    else dict(node, **{"color": "#D9651D"})
    if node["id"] == "CytoTable"
    else node
    for node in nt_dependents.nodes
]
nt_dependents.toggle_physics(True)
nt_dependents.html_export_loc = "target-project-dependents.html"
nt_dependents.html_plot_title = "Project dependents Graph"
nt_dependents.show(nt_dependents.html_export_loc)

fig_collection.append(
    {
        "plot": nt_dependents,
        "description": """This graph shows how pycytominer and CytoTable are used as
        dependencies of existing projects on GitHub (from Dependency Graph and code search by name).
        Pycytominer is included as a pink node with edges in the same color. CytoTable is shown similarly in orange.
        """,
        "findings": """We observe pycytominer as being connected to many existing projects.
        CytoTable is also connected to many Pycytominer projects. This interconnection in part shows
        how these components are used together in related landscape projects.
            """,
        "section": "Software dependents",
    }
)

IFrame(src=nt_dependents.html_export_loc, height=520, width=1110)

# +
# alternative network graph for dependents
dependent_data = pd.concat(
    pd.DataFrame(
        [
            {
                "Project Name": project_name,
                "dependent": dependent.replace("cytomining/CytoTable", "CytoTable"),
                "color": "#E7358A"
                if project_name == "pycytominer"
                else "#D9651D"
                if project_name == "CytoTable"
                else "#698DC7",
            }
            for dependent in list(
                set(
                    code_search
                    + [
                        repo["name"]
                        for repo in graph_results["all_public_dependent_repos"]
                    ]
                )
            )
        ]
    )
    for project_name, code_search, graph_results in zip(
        tgt_software_df["Project Name"],
        tgt_software_df["GitHub Code Search Used By"],
        tgt_software_df["GitHub Dependents"],
    )
)

dependent_data

nx_dependent_data = nx.from_pandas_edgelist(
    df=dependent_data, source="dependent", target="Project Name", edge_attr=["color"]
)

nx.set_node_attributes(
    nx_dependent_data,
    values={
        node: "#E7358A"
        if node == "pycytominer"
        else "#D9651D"
        if node == "CytoTable"
        else "#698DC7"
        for node in nx_dependent_data.nodes
    },
    name="group",
)

"""stylized_network, config = nw.visualize(
    network=nx_dependent_data,
    config={"zoom": 2},
    is_test=True,
    plot_in_cell_below=False,
)"""

# +
# form monthly github star visual
star_monthly_data = pd.concat(
    pd.DataFrame(
        [
            dict(month_entry, **{"Project Name": project_name})
            for month_entry in [
                {"star_month": key, "star_count": val}
                for key, val in monthly_data.items()
            ]
        ]
    )
    for project_name, monthly_data in zip(
        tgt_software_df["Project Name"],
        tgt_software_df["GitHub Stargazers Count by Month"],
    )
    if isinstance(monthly_data, dict)
)
fig_stars_monthly = px.line(
    data_frame=star_monthly_data,
    title=f"Project GitHub Stars by Month",
    x="star_month",
    y="star_count",
    color="Project Name",
    markers=True,
    width=1200,
    height=500,
    symbol_sequence=["star"],
    color_discrete_sequence=project_color_sequence,
)
fig_stars_monthly.update_layout(
    xaxis_title="Month",
    yaxis_title="Stars",
)

fig_collection.append(
    {
        "plot": fig_stars_monthly,
        "description": """This plot shows GitHub stars as a way to understand visitor activity on a
        monthly basis for the target projects.
        GitHub stars are an indication of unique users whom are interested in and likely use the project somehow.
        """,
        "findings": """We can observe pycytominer has having a relatively consistent number of added GitHub stars
        since mid-2019. We also show that CytoSnake and CytoTable somewhat mimic the earlier growth in this area
        which pycytominer demonstrated.
        """,
        "section": "Project Visitors",
    }
)

fig_stars_monthly.show()

# +
# form cumulative sum outlook of the monthly star data
star_monthly_data["star_count_cumulative_sum"] = star_monthly_data.groupby(
    "Project Name"
)["star_count"].transform(pd.Series.cumsum)

fig_stars_monthly_cumulative_sum = px.line(
    data_frame=star_monthly_data,
    title=f"Project GitHub Stars by Month (cumulative)",
    x="star_month",
    y="star_count_cumulative_sum",
    color="Project Name",
    markers=True,
    width=1200,
    height=500,
    symbol_sequence=["star"],
    color_discrete_sequence=project_color_sequence,
)
fig_stars_monthly_cumulative_sum.update_layout(
    xaxis_title="Month",
    yaxis_title="Stars (cumulative)",
)

fig_collection.append(
    {
        "plot": fig_stars_monthly_cumulative_sum,
        "description": """This plot shows GitHub stars as a way to understand visitor activity on a
        monthly basis for the target projects.
        This plot differs from the one above in that the star counts are shown as cumulative.
        GitHub stars are an indication of unique users whom are interested in and likely use the project somehow.
        """,
        "findings": """We can observe pycytominer has having a relatively consistent number of added GitHub stars
        since mid-2019. We also show that CytoSnake and CytoTable somewhat mimic the earlier growth in this area
        which pycytominer demonstrated.
        """,
        "section": "Project Visitors",
    }
)

fig_stars_monthly_cumulative_sum.show()
# -

# organize figures by their sections and the order in which they appeared in this notebook
fig_collection_grouped = {
    key: [
        {
            "plot": entry["plot"],
            "description": entry["description"],
            "findings": entry["findings"],
        }
        for entry in group
    ]
    for key, group in itertools.groupby(fig_collection, key=lambda x: x["section"])
}
fig_collection_grouped.keys()

cdn_included = False
with open(f"{export_dir}/target-project-report.html", "w") as f:
    f.write(
        """

    <title>Cytomining Ecosystem | Way Lab: Software Landscape Analysis</title>
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <meta charset="UTF-8">
    <link rel="stylesheet" href="../css/style.css">

Cytomining Ecosystem Software Landscape Analysis - Target Project Report

This is a targeted report to help describe three Cytomining Ecosystem software projects
        in terms of scholarly paper mentions, package usage, software dependents, and project visitors.

Code related to this effort may be found at: https://github.com/WayScience/software-landscape-analysis.

    """
    )

    for section, figures in fig_collection_grouped.items():
        f.write(f"
{section}
")
        f.write(f"
{section_descriptions[section]}
")
        f.write(f"

")
        for figure in figures:
            # if working with plotly figure
            if isinstance(figure["plot"], plotly.graph_objs._figure.Figure):
                f.write(
                    figure["plot"].to_html(
                        full_html=False,
                        include_plotlyjs="cdn" if not cdn_included else False,
                    )
                )
            # if working with pyvis graph
            elif isinstance(figure["plot"], pyvis.network.Network):
                # write the network plot to docs dir
                figure["plot"].show(f"{export_dir}/{figure['plot'].html_export_loc}")

                # copy the dependencies of the work to docs dir
                try:
                    shutil.copytree(
                        src="./lib", dst=f"{export_dir}/lib", dirs_exist_ok=True
                    )
                except:
                    continue

                # use an iframe to display the result within the same page as other figures
                f.write(
                    f"""

                    {figure['plot'].html_plot_title} (click and scroll with mouse to interact)
                    Browser not compatible with iframe rendering.

                    """
                )

                # correct the file for invalid html
                def fix_html(
                    file_path,
                    strs_to_replace=[
                        "

",
                        "",
                        "",
                    ],
                ):
                    with open(file_path, "r") as file:
                        lines = [
                            re.sub(
                                "|".join(map(re.escape, strs_to_replace)),
                                "",
                                line.replace(
                                    "",
                                    "<title>Cytomining Ecosystem Target Project Dependents Graph</title>",
                                ),
                            )
                            for replace_str in strs_to_replace
                            for line in file
                        ]

                    with open(file_path, "w") as file:
                        file.writelines(lines)

                fix_html(f"{export_dir}/{figure['plot'].html_export_loc}")

            else:
                raise Exception("Unknown plot type used.")

            f.write(
                f"""

Description: {figure['description']}

Findings: {figure['findings']}

            """
            )
            cdn_included = True

    f.write(
        """

Table with selected dataset columns

The table below may be used to search and view a selected number of columns from the dataset.

        """
    )

    # rename columns for visibility
    tgt_software_df = tgt_software_df.rename(
        columns={
            "google_scholar_count": "Google Scholar Count",
            "biorxiv_count": "bioRxiv Count",
            "pypi_downloads_total": "PyPI Downloads Total",
            "pypi_downloads_monthly_average": "PyPI Downloads Monthly Avg",
            "conda_downloads_total": "Conda Downloads Total",
            "conda_downloads_monthly_average": "Conda Downloads Monthly Avg",
        }
    )

    # write an itable to the page
    f.write(
        to_html_datatable(
            tgt_software_df[
                [
                    "Project Name",
                    "Date Created Year",
                    "Google Scholar Count",
                    "bioRxiv Count",
                    "PyPI Downloads Total",
                    "PyPI Downloads Monthly Avg",
                    "Conda Downloads Total",
                    "Conda Downloads Monthly Avg",
                    "GitHub Stars",
                    "GitHub Contributor Total Count",
                    "GitHub Total Dependents Count",
                ]
            ],
            style="height:600px;float:left;",
            classes="display",
            maxBytes=0,
            # fix a malformed attributes error built into the html content
        ).replace('class="display"style="', 'class="display" style="')
    )
    f.write("
")

# +
# capture the html page as a png export as a backup

# allow for nested asyncio ops
nest_asyncio.apply()

# define function for pyppeteer to capture image
# see: https://github.com/pyppeteer/pyppeteer#examples
async def capture_screenshot(file_path, output_path):
    browser = await launch(headless=True)
    page = await browser.newPage()
    # set the size of the capture
    await page.setViewport({"width": 1400, "height": 6500})
    await page.goto(f"file://{file_path}")
    await page.waitFor(1000)
    await page.screenshot({"path": output_path})
    await browser.close()

# Capture screenshot from html page
asyncio.get_event_loop().run_until_complete(
    capture_screenshot(
        file_path=pathlib.Path(f"{export_dir}/target-project-report.html").resolve(),
        output_path=f"{export_dir}/target-project-report.png",
    )
)


import http
import os
from base64 import b64encode
from typing import Dict
from urllib.parse import quote

import requests
from requests import Request, Session
from requests.adapters import HTTPAdapter
from requests.auth import CONTENT_TYPE_FORM_URLENCODED
from yarl import URL

class BasicAuth(requests.auth.AuthBase):
    def __init__(self, token: str):
        self.token = token

    def __call__(self, r: Request):
        r.headers["Authorization"] = "Basic " + self.token
        return r

class Admitad:
    API_URL = URL("https://api.admitad.com/")
    TOKEN_PATH = "token/"
    AUTH_PATH = "authorize/"
    DEEPLINK_URL = "deeplink/{website_id}/advcampaign/{campaign_id}/"
    COUPONS_URL = "coupons/"
    COUPON_URL = "coupons/{id}/"

    def __init__(self, client_id, client_secret: str, scopes: str):
        self.session = Session()
        self.session.mount(str(self.API_URL), HTTPAdapter(max_retries=5))
        user_data = self.get_oauth_token(client_id, client_secret, scopes)
        self.session.headers.update(
            {
                "Authorization": "Bearer " + user_data.get("access_token"),
                "Connection": "Keep-Alive",
            }
        )

    @staticmethod
    def encode_credentials(client_id, client_secret: str):
        return b64encode(
            "{}:{}".format(client_id, client_secret).encode()
        ).decode("utf-8")

    def get_oauth_token(self, client_id, client_secret, scopes: str):
        params = {
            "grant_type": "client_credentials",
            "client_id": client_id,
            "scope": scopes,
        }
        credentials = self.encode_credentials(client_id, client_secret)
        headers = {
            "Content-Type": CONTENT_TYPE_FORM_URLENCODED,
        }
        response = self.session.post(
            str(self.API_URL / self.TOKEN_PATH),
            data=params,
            headers=headers,
            auth=BasicAuth(credentials),
        )
        if response.status_code != http.HTTPStatus.OK:
            raise BaseException("Bad status code ", response.status_code)
        return response.json()

    def get_coupons(self, params: Dict):
        response = self.session.get(
            str(self.API_URL / self.COUPONS_URL), params=params
        )
        if response.status_code != http.HTTPStatus.OK:
            raise BaseException("Bad status code ", response.status_code)
        data = response.json()
        return data

    def generate_deeplink(self, website_id, campaign_id: int, ulp, subid: str):
        response = self.session.get(
            str(
                self.API_URL / self.DEEPLINK_URL.format(
                    website_id=website_id, campaign_id=campaign_id
                )
            ),
            params={"ulp": quote(ulp), "sibod": subid},
        )
        if response.status_code != http.HTTPStatus.OK:
            raise BaseException("Bad status code ", response.status_code)
        data = response.json()
        return data

if __name__ == "__main__":
    admitad = Admitad(
        os.getenv("CLIENT_ID"),
        os.getenv("CLIENT_SECRET"),
        "advcampaigns banners websites coupons deeplink_generator",
    )
    # тут получаем купон в c типом промокод, но в ответе промокода нет
    # нужно ли как-то отдельно получать промокод?
    coupons = admitad.get_coupons({"type": 1, "limit": 1, "region": "RU"})
    print(coupons.get("results"))

    # пытаемся сгенерить диплинку по доке
    # https://developers.admitad.com/ru/doc/api_ru/methods/deeplink/deeplink/
    # получаем 404
    admitad.generate_deeplink(
        22, 10, "https://admitad.com/some/", "AS32djkd31"
    )


from _sha256 import sha256
from typing import List

from common.serializers.json_serializer import JsonSerializer
from plenum.common.event_bus import InternalBus, ExternalBus
from plenum.common.messages.node_messages import ViewChange, ViewChangeAck, NewView
from plenum.common.timer import TimerService
from plenum.server.consensus.consensus_data_provider import ConsensusDataProvider
from plenum.server.quorums import Quorums

class ViewChangeVotesForView:
    """
    This is a container for view change votes for specific view
    """
    def __init__(self, quorums: Quorums):
        self._quorums = quorums

    @property
    def has_new_view_quorum(self) -> bool:
        return False

    def add_view_change(self, msg: ViewChange, frm: str):
        pass

    def add_view_change_ack(self, msg: ViewChangeAck, frm: str):
        pass

class ViewChangeService:
    def __init__(self, data: ConsensusDataProvider, timer: TimerService, bus: InternalBus, network: ExternalBus):
        self._data = data
        self._timer = timer
        self._bus = bus
        self._network = network

        network.subscribe(ViewChange, self.process_view_change_message)
        network.subscribe(ViewChangeAck, self.process_view_change_ack_message)
        network.subscribe(NewView, self.process_new_view_message)

    def start_view_change(self):
        # TODO: Calculate
        prepared = []
        preprepared = []

        self._data.view_no += 1
        self._data.waiting_for_new_view = True
        self._data.primary_name = self._find_primary(self._data.validators, self._data.view_no)

        vc = ViewChange(
            viewNo=self._data.view_no,
            stableCheckpoint=self._data.stable_checkpoint,
            prepared=prepared,
            preprepared=preprepared,
            checkpoints=self._data.checkpoints
        )
        self._network.send(vc)

    def process_view_change_message(self, msg: ViewChange, frm: str):
        # TODO: Validation

        # TODO: Probably we should stash messages from future view instead?
        if not self._is_primary(msg.viewNo):
            vca = ViewChangeAck(
                viewNo=msg.viewNo,
                name=frm,
                digest=self._view_change_digest(msg)
            )
            self._network.send(vca, self._data.primary_name)

    def process_view_change_ack_message(self, msg: ViewChangeAck, frm: str):
        # TODO: Validation
        if msg.viewNo != self._data.view_no:
            return

        if not self._data.is_primary:
            return

        nv = NewView(
            viewNo=msg.viewNo,
            viewChanges=[],
            checkpoint=None,
            preprepares=[]
        )
        self._network.send(nv)
        self._data.waiting_for_new_view = False

    def process_new_view_message(self, msg: NewView, frm: str):
        # TODO: Validation

        self._data.waiting_for_new_view = False

    @staticmethod
    def _view_change_digest(msg: ViewChange) -> str:
        serialized = JsonSerializer().dumps(msg.__dict__)
        return sha256(serialized).hexdigest()

    @staticmethod
    def _find_primary(validators: List[str], view_no: int) -> str:
        return validators[view_no % len(validators)]

    def _is_primary(self, view_no: int) -> bool:
        # TODO: Do we really need this?
        return self._find_primary(self._data.validators, view_no) == self._data.name


import pandas as pd

from celescope.tools.capture.analysis import Analysis, get_opts_analysis
from celescope.fusion.count_fusion import Count_fusion
from celescope.fusion.mkref import Mkref_fusion
from celescope.tools.plotly_plot import Tsne_dropdown_plot,Tsne_single_plot,Tsne_plot

def analysis_fusion(args):

    with Analysis_fusion(args) as runner:
        runner.run()

def get_opts_analysis_fusion(parser, sub_program):
    parser.add_argument('--fusion_genomeDir', help='Fusion genome directory.', required=True)
    get_opts_analysis(parser, sub_program)

class Analysis_fusion(Analysis):
    def __init__(self, args, display_title='Analysis'):
        super().__init__(args, display_title)

        fusion_pos_file =  Mkref_fusion.get_config(args.fusion_genomeDir)['files']['fusion_pos']
        self.pos_dict = Count_fusion.read_pos_file(fusion_pos_file)

        self.count_fusion_df = None
        self.count_fusion_out = f'{self.out_prefix}_fusion_count.csv'

    def plot_fusion(self):
        """
        plot fusion count
        """
        df_tsne_file = f'{self.out_prefix}_UMI_tsne.csv'
        df_tsne = pd.read_csv(df_tsne_file,sep=",",index_col=0)
        feature_name_list = df_tsne.columns[4:-1].to_list()
        # plot
        tsne_cluster = Tsne_plot(df_tsne, 'cluster').get_plotly_div()
        self.add_data(tsne_cluster=tsne_cluster)
        tsne_citeseq = Tsne_dropdown_plot(df_tsne,'Fusion',feature_name_list).get_plotly_div()
        self.add_data(tsne_citeseq=tsne_citeseq)
        Tsne_single_plot(df_tsne,feature_name_list,self.args.outdir).get_plotly_div()

    def get_fusion_count_df(self):
        fusion_df = self.df_tsne.reset_index().filter(list(self.pos_dict.keys()) + ["barcode"])
        fusion_df = fusion_df.melt(id_vars=["barcode"],value_name="UMI",var_name="fusion",ignore_index=True)

        count = fusion_df[fusion_df["UMI"] > 0].groupby(["fusion"], as_index=False)["barcode"].count()
        self.count_fusion_df = count.assign(percent = lambda x: 100 * x.barcode / len(self.df_tsne))
        self.count_fusion_df.to_csv(self.count_fusion_out, index = None)

    def add_fusion_count_metrics(self):
        self.add_help_content('fusion_1, fusion_2...', 'number of positive cells(UMI > 0) of each fusion')
        for _, row in self.count_fusion_df.iterrows():
            self.add_metric(
                name=f'{row["fusion"]}',
                value=int(row['barcode']),
                total=int(len(self.df_tsne)),
            )

    def run(self):
        super().run()
        self.plot_fusion()
        self.get_fusion_count_df()
        self.add_fusion_count_metrics()


import requests
from TOKEN import TOKEN

name_papka_unittest = 'папка_для_unittests'
name_papka_pytest = 'папка_для_pytest'

def header(TOKEN):
    headers = {'Content-Type': 'application/json',
            'Authorization': f'OAuth {TOKEN}'
            }
    return headers

def create(name_papka):
    headers = header(TOKEN)
    url = 'https://cloud-api.yandex.net/v1/disk/resources/'
    params = {'path': name_papka}
    response1 = requests.put(url, headers=headers, params = params)
    return response1.status_code

# create(name_papka)


from django.urls import path

from solf.apps.classes.api import views

urlpatterns = [
    path(
        "",
        views.ClassViewSet.as_view({"get": "list"}),
        name="class-list",
    ),
    path(
        "<int:pk>/",
        views.ClassViewSet.as_view(
            {
                "get": "retrieve",
            }
        ),
        name="class-details",
    ),
    path(
        "schedule/",
        views.ClassScheduleViewSet.as_view({"get": "list"}),
        name="class-schedule-list",
    ),
    path(
        "schedule/<int:pk>/",
        views.ClassScheduleViewSet.as_view(
            {
                "get": "retrieve",
            }
        ),
        name="class-schedule-details",
    ),
    path(
        "schedule/<int:pk>/sign-up/",
        views.ClassScheduleViewSet.as_view(
            {
                "post": "sign_up",
            }
        ),
        name="class-schedule-sign-up",
    ),
    path(
        "logs/",
        views.ClassLogViewSet.as_view({"get": "list"}),
        name="class-logs-list",
    ),
    path(
        "logs/<int:pk>/",
        views.ClassLogViewSet.as_view(
            {
                "get": "retrieve",
            }
        ),
        name="class-logs-details",
    ),
    path(
        "entries/my",
        views.ClassEntryViewSet.as_view({"get": "list"}),
        name="class-logs-list",
    ),
]


import time
import datetime

#run    "pip install win10toast"     if not done already
from win10toast import ToastNotifier

toaster = ToastNotifier()

toaster.show_toast("Toast Reminders","This is how you will be reminded",
                    icon_path="i.ico")

data = []
for d in open("times").read().split("\n"):
    if len(d.split("~")) == 3:
        data.append(d.split("~"))

while True:
    time.sleep(20)
    if datetime.datetime.today().weekday() < 5: # is a weekday (mon=0, sun=6)
        now = datetime.datetime.now()
        current_time = now.strftime("%H:%M")
        for t in data:
            if current_time == t[0]:  #if the time is the same, then show the notification
                toaster.show_toast(
                    t[1],
                    t[2],
                    icon_path="i.ico",
                    duration=60,
                    threaded=True
                    )
                time.sleep(70) # will mean that it never notifies more than needed (it may notify many times within the minute otherwise)


"""
A paint-style 2D toy dataset builder.
The user clicks on the canvas to create groups of normally distributed points in one of the
3 available colors. The canvas can be cleared, or the last point undone.
Dataset is scaled to zero mean and unit variance before saving.
The dataset created can be saved to a .csv file.
"""

import Tkinter as tk
import tkFileDialog
from numpy.random import normal
import pandas as pd
from sklearn import preprocessing

class Dataset:
    """
    A class managing the data created by the user and displaying it on a Tkinter canvas.
    """

    def __init__(self, drawingArea, sigmaScatter, nbrPointsScatter):
        """
        Initialize with an empty dataset.
        Args:
          drawingArea (tk.Canvas): canvas for displaying the dataset
          sigmaScatter (tk.DoubleVar): spread of a group of points created by a click
          nbrPointsScatter (tk.DoubleVar): number of points per click group
        """
        # myData is a list of coordinates and color for each point in the dataset
        self.myData = []
        # tkinter IDs for each data point
        self.myPointsID = []
        # marker size for the point
        self.sizePoint = 2
        # Current class (color) attributed to next points being drawn ("blue", "green", "red")
        self.classPoint = "blue"
        self.drawingArea = drawingArea
        self.sigmaScatter = sigmaScatter
        self.nbrPointsScatter = nbrPointsScatter

    def setClass(self, color):
        """
        Change point color
        Args:
          color (String): color for the next points that will be created
        """
        self.classPoint = color

    def makeScatter(self, event):
        """
        Draw points sampled from a gaussian distribution around the click point
        Args:
          event (tk.Event): tk user click event
        """
        # clamp values to a reasonable interval
        ss = max(min(self.sigmaScatter.get(), 50), 0)
        ps = max(min(self.nbrPointsScatter.get(), 1000), 0)
        listX = normal(loc=event.x, scale=ss, size=ps)
        listY = normal(loc=event.y, scale=ss, size=ps)
        for (x,y) in zip(listX, listY):
            self.myData.append([x, y, self.classPoint])
            newpoint = event.widget.create_oval(x-self.sizePoint, y-self.sizePoint,
                                                x+self.sizePoint, y+self.sizePoint,
                                                fill=self.classPoint)
            self.myPointsID.append(newpoint)

    def saveData(self):
        """
        Save the x,y,class properties of each drawn points to a csv file.
        Dataset is scaled to zero mean and unit variance before saving.
        """
        saveTo = tkFileDialog.asksaveasfile(mode='w', defaultextension="")
        if saveTo is None:
            return None
        df = pd.DataFrame(self.myData, columns = ["x","y","color"])
        #scale to zero mean and unit variance
        df[['x','y']] = df[['x','y']].apply(lambda x: preprocessing.scale(x))
        df.to_csv(saveTo.name, index=False)
        saveTo.close()

    def clearCanvas(self):
        """
        Remove all points on canvas.
        """
        for pointID in self.myPointsID:
            self.drawingArea.delete(point)
        self.myPointsID = []
        self.myData = []

    def undo(self):
        """
        Remove the last drawn point. Raises exception if no points are defined.
        """
        self.myData.pop()
        self.drawingArea.delete(self.myPointsID.pop())

if __name__ == "__main__":

    # build TK window
    root = tk.Tk()
    root.wm_title("Draw Dataset")
    # for the points
    drawingArea = tk.Canvas(root, width=600, height=600)
    # for the parameters
    frameVars = tk.Frame(root, bd=2)

    # drawing parameters default values
    sigmaScatter = tk.DoubleVar(value='10')
    nbrPointsScatter = tk.DoubleVar(value='5')

    # create display for parameter edition
    tk.Label(frameVars, text='stdDEV').pack(side=tk.LEFT)
    esigma = tk.Entry(frameVars,
                      textvariable = sigmaScatter).pack(side=tk.LEFT,padx=3)
    tk.Label(frameVars, text='nbrPoints').pack(side=tk.LEFT)
    escatter = tk.Entry(frameVars,
                        textvariable = nbrPointsScatter).pack(side=tk.LEFT,padx=3)

    # create windows
    drawingArea.create_window(1, 1, anchor=tk.NW, window=frameVars)
    frameVars.pack(anchor=tk.NW)
    drawingArea.pack()

    data = Dataset(drawingArea, sigmaScatter, nbrPointsScatter)

    # bind mouse click to point drawing
    drawingArea.bind("<Button>", data.makeScatter)

    # button positioning parameters
    width = 5
    sep = 60
    margin = 5

    # point color buttons
    bcolorBlue = tk.Button(root, text="blue", command=lambda: data.setClass("blue"), width=width)
    drawingArea.create_window(margin, 10, anchor=tk.NW, window=bcolorBlue)

    bcolorRed = tk.Button(root, text="red", command=lambda: data.setClass("red"), width=width)
    drawingArea.create_window(margin+sep, 10, anchor=tk.NW, window=bcolorRed)

    bcolorGreen = tk.Button(root, text="green", command=lambda: data.setClass("green"), width=width)
    drawingArea.create_window(margin+2*sep, 10, anchor=tk.NW, window=bcolorGreen)

    # dataset actions buttons
    bsave = tk.Button(root, text="save", command=data.saveData, width=width)
    drawingArea.create_window(margin+3*sep, 10, anchor=tk.NW, window=bsave)

    bclear = tk.Button(root, text="clear", command=data.clearCanvas, width=width)
    drawingArea.create_window(margin+4*sep, 10, anchor=tk.NW, window=bclear)

    bundo = tk.Button(root, text="undo", command=data.undo, width=width)
    drawingArea.create_window(margin+5*sep, 10, anchor=tk.NW, window=bundo)

    tk.mainloop()


from datetime import datetime
from random import choice

from sqlalchemy import and_, or_

from shorter.database import Model
from shorter.start.environment import DELAY_DEF, SYM_MINI, SYM_POOL
from shorter.start.extensions import DB
from shorter.start.helper import parse_int
from shorter.support import BlocklistValidator

# pylint: disable=no-member

class Short(Model):
    symbol = DB.Column(DB.String(length=256), unique=True, nullable=False)
    target = DB.Column(DB.String(length=1024), nullable=False)
    delay = DB.Column(DB.Integer(), nullable=False, default=DELAY_DEF)
    active = DB.Column(DB.Boolean(), nullable=False, default=True)
    visited = DB.Column(DB.Integer(), nullable=False, default=0)
    created = DB.Column(DB.DateTime(), nullable=False, default=datetime.utcnow)

    @classmethod
    def by_symbol(cls, symb):
        return cls.query.filter(cls.symbol == symb).first()

    @classmethod
    def by_symbol_active(cls, symb):
        result = cls.by_symbol(symb)
        if not result:
            return None
        if not result.active:
            return None
        return result

    @staticmethod
    def generate_symbol(length):
        return "".join(choice(SYM_POOL) for _ in range(1, 1 + abs(length)))

    @classmethod
    def len_symbol(cls, minimum=SYM_MINI):
        result = abs(minimum if minimum != 0 else 1)
        count = cls.query.count()
        size = len(SYM_POOL)
        while pow(size, result) <= count:
            result += 1
        return result

    @classmethod
    def make_symbol(cls, minimum=SYM_MINI):
        length = cls.len_symbol(minimum=minimum)
        result = cls.generate_symbol(length)
        while cls.by_symbol(result) is not None:
            result = cls.generate_symbol(length)
        return result

    @classmethod
    def generate(cls, target, delay=DELAY_DEF, _commit=True, **kwargs):
        short = cls.query.filter(
            and_(
                cls.target == target,
                cls.delay == delay,
                cls.active.is_(True),
            )
        ).first()
        if short is not None:
            return short

        return cls.create(
            symbol=cls.make_symbol(minimum=SYM_MINI),
            target=target,
            delay=delay,
            _commit=_commit,
            **kwargs,
        )

    def increase_visit(self, _commit=True):
        value = parse_int(self.visited)
        return self.update(visited=1 + value, _commit=_commit)

    @classmethod
    def ordered(cls, field, *, rev=False, query=None):
        field = field if field is not None else "prime"
        query = query if query is not None else cls.query

        f_coll = cls.__table__.columns.get(field, None)
        p_coll = cls.__table__.columns.get("prime", None)
        if f_coll is None or p_coll is None:
            return None

        return query.order_by(
            f_coll.desc() if rev else f_coll.asc(),
            p_coll.desc() if rev else p_coll.asc(),
        )

    @classmethod
    def searched(cls, term, *, field=None, rev=None, query=None):
        query = query if query is not None else cls.query
        query = query.filter(
            or_(
                Short.symbol.like(f"%{term}%"),
                Short.target.like(f"%{term}%"),
            )
        )

        return cls.ordered(field=field, rev=rev, query=query)

    @classmethod
    def blocked(cls, blocklist, *, field=None, rev=None, query=None):
        query = query if query is not None else cls.query
        validator = BlocklistValidator(blocklist)
        query = query.filter(
            Short.prime.in_(validator.prime_targets(cls.query.all()))
        )

        return cls.ordered(field=field, rev=rev, query=query)


from default.framework.request_handler import SearchRequestHandler
from google.appengine.api import taskqueue

class LoadURLs(SearchRequestHandler):
    def get(self):
        queue = taskqueue.Queue('parser')

        with open('modules/parser/src/food_com_small.txt', 'r') as list_urls:
            for url in list_urls:
                task = taskqueue.Task(url='/_parser/parse', params={'url': url.strip()})
                queue.add(task)
class Parse(SearchRequestHandler):
    def post(self):
        url = self.request.get('url')


from .client import VSphereClient

from pyVmomi import vim

class VSphereResource(object):

    def __init__(self, config):
        self.client = VSphereClient().create_client(config)
        self.content = self.client.RetrieveContent()

    def get_instances(self, params):

        vim_type = [vim.VirtualMachine]

        container_view = self.content.viewManager.CreateContainerView(
            self.content.rootFolder, vim_type, True)
        objects = container_view.view
        container_view.Destroy()
        res = []
        for server_id in params:
            for obj in objects:
                if server_id == obj._moId:
                    instance = {'id': str(obj._moId),
                                'name': str(obj.name),
                                'status': str(obj.guest.guestState)}
                    res.append(instance)

        return res


acumular = int()
acumular2 = int()
k = int(input())
producte = 1
if k > 0:
    for valors in range (k):
        n = int(input())
        if n > 0:
            for valor in range (n):
                n += valor
                producte *= valor+1
            print ("SUMA: ",n,"PRODUCTE: ",producte)
            producte = 1
        else:
            print("ELS NOMBRES NATURALS COMENCEN EN 1")

UF1/PROVES/Bucles 1 - Producte números naturals.py


"""Classe de template."""
import sys

from .factory import Factory
from .sh import Sh

class TProjects__others__ezored__src__maindot_cpp(Factory):
    """Classe de template."""

    def __init__(self, data_model, template_name):

        Factory.__init__(self, template_name)
        self.data_model = data_model
        self.tmpl.data = data_model["data"]
        self.tmpl.name = "%s/%s" % (
            self.tmpl.data["dir"],
            "projects/others/ezored/src/main.cpp".replace(
                "ezored", self.tmpl.data["project"]["name"]
            ),
        )

    def put(self):
        fileName = "%s" % self.tmpl.name
        print("File: %s" % fileName, file=sys.stderr)
        print(self.tmpl)

    def save(self):
        fileName = "%s" % self.tmpl.name
        sh = Sh(fileName)
        sh.mkdir()
        print("Save File: %s" % fileName)
        open(fileName, "w").write(str(self.tmpl))


import numpy as np
import pandas as pd
from sklearn.linear_model import LogisticRegression
import matplotlib.pyplot as plt
from sklearn.preprocessing import StandardScaler
df_train=pd.read_csv("/Users/ningduan/Python/MachineLearning/LogisticRegression/heart_train.csv")
df_test=pd.read_csv("/Users/ningduan/Python/MachineLearning/LogisticRegression/heart_test.csv")
#print(df_train)
#target would be y
#input dataset
X_train=df_train[df_train.columns[:-1]].to_numpy()
X_test=df_test[df_test.columns[:-1]].to_numpy()
#print(X_train)
y_train=df_train[df_train.columns[-1]].to_numpy()
y_test=df_test[df_test.columns[-1]].to_numpy()
#print(y_train)
#standscaler
ss = StandardScaler()# the same thing with z-score transformation
X_train_z = ss.fit_transform(X_train)
#X_test_ss = ss.fit_transform(X_test)
#print(X_train)
X_train_mean = X_train.mean(axis=0)#select object across rows, vertically
X_train_std = X_train.std(axis=0)
# X_train_z=(X_train-X_train_mean)/X_train_std
X_test_z=(X_test-X_train_mean)/X_train_std #we are not using scaler for test dataset, we want use regular z-score calculation
print(X_test_z)
lr=LogisticRegression(max_iter=10000,penalty="none")
lr.fit(X_train_z,y_train)

#lr.fit(X_test,y_test)
#print()
#print("Train R^2: %f"%lr.score(X_train, y_train))
#print("Test R^2:  %f"%lr.score(X_test, y_test))
#print()
#print("Coefficients:")
#print(list(zip(df_train.columns, lr.coef_)))

#heatmap
fig=plt.figure()
ax=fig.add_axes([0.1,0.2,0.8,0.9])
cmax=np.abs(lr.coef_).max()
cax=ax.matshow(np.reshape(lr.coef_,[-1,1]),cmap='seismic',aspect='1',vmax=cmax,vmin=-1.0*cmax)
fig.colorbar(cax)
ax.set_yticks(list(range(len(df_train.columns[:-1]))))
ax.set_yticklabels(df_train.columns[:-1])
ax.set_xticks([])
ax.set_xlabel('Logistic Regression Betas')
#plt.show()


class Solution:
    #-1 to sell 0 to buy 1 to cool down -2 to freedom
    def maxProfit(self, prices: List[int]) -> int:
        memo=dict()
        def helper(n,stat):
            if n>len(prices)-1:
                return 0
            elif (n,stat) in memo:
                return memo[(n,stat)]
            else:
                if stat==0:
                    buy=-prices[n]+helper(n+1,1)
                    buy_not=helper(n+1,0)
                    memo[(n,stat)]=max(buy,buy_not)
                elif stat==1:
                    sell=prices[n]+helper(n+2,0)
                    sell_not=helper(n+1,1)
                    memo[(n,stat)]=max(sell,sell_not)
                return memo[(n,stat)]
        return helper(0,0)

Best Time to Buy and Sell Stock with Cooldown.py


# AUTHOR: LUIS FERNANDO

import pandas as pd
from copy import deepcopy
import os
import sys
import time
import pickle

# Recording initial time of execution
start_1 = time.time()

# 0.a) Open the managing files:
df_mngr_overall = "data_manager_overall.xlsx"
df_country_2_reg = pd.read_excel(df_mngr_overall, sheet_name="country_2_reg")
df_mngr = "data_manager_olade_iea_other.xlsx"

# 0.b) Open the input files:
dir_caps = './data_input_processing/olade_capacities'
caps_filename = dir_caps + '/' + 'all_countries_capacity.xlsx'

# 0.c) A dictionary is required to align the gathered thermal power plant data
### dict_thermal_equiv = {
###    'Fuel oil':['Fuel Oil/Bunker', 'Petróleo/Crudo']}

dict_thermal_equiv = {
    'Fuel oil':['Fuel Oil/Bunker'],
    'Crude':['Petróleo/Crudo'],
    'Diesel':['Diesel'],
    'Natural Gas':['Gas Natural'],
    'Coal':['Carbón']}

dir_thermal_caps = './data_input_processing/thermal_capacities'
thermal_caps_file = dir_thermal_caps + '/' + 'thermal_cap_distribution.xlsx'
df_therm_cap = pd.read_excel(thermal_caps_file, sheet_name="2021")

hyear_vector = [
    2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021]
hyear_vector_sh = [2018, 2019, 2020, 2021]

# Control which elements you want to process:
process_energy_balances = True
process_electrical_capacities = True
process_electrical_capacities_raw = True

# Control how you want to process the data:
thermal_share_manual_mode = False
thermal_share_automatic = True

###############################################################################
if process_energy_balances is True:
    # 1) First, let's process the energy balance databases
    print('1) Processing energy balance data')
    dir_balances = './data_input_processing/energy_balances'

    # Objective: create a nested dictionary of region/country/variable/set/year
    # Crucially, variable <=> balance rows && set <=> balance cols

    dict_db = {}
    wide_source = []
    wide_file = []
    wide_region = []
    wide_country = []
    wide_variable_main = []
    wide_variable_disag = []
    wide_set = []
    wide_year = []
    wide_value = []

    # Open the nesting and labeling sheets from control:
    df_control_EB_nest = pd.read_excel(df_mngr, sheet_name="EB_nest")
    df_control_EB_labeling = pd.read_excel(df_mngr, sheet_name="EB_labeling")
    list_EB_nest_main = df_control_EB_nest['main'].tolist()
    list_EB_nest_disag = df_control_EB_nest['disag'].tolist()
    df_EB_label_name = df_control_EB_labeling['name'].tolist()
    df_EB_labels = df_control_EB_labeling['label'].tolist()

    list_regions = list(set(df_country_2_reg['Region'].tolist()))
    list_regions.sort()

    all_lac_country_list = list(set(df_country_2_reg['Country'].tolist()))
    country_2_region = {}
    for index, row in df_country_2_reg.iterrows():
        a_country = row['Country']
        a_region = row['Region']
        country_2_region[a_country] = a_region

    list_of_balances = os.listdir(dir_balances)
    list_of_balances_filter = [i for i in list_of_balances if '$' not in i]
    """
    Note 1: the name of the balances must be equal to the names of countries
    in the "df_country_2_reg" dataframe.
    """

    list_of_countries = [
        a_con.replace('.xlsx', '') for a_con in list_of_balances]

    for reg in list_regions:  # iterate across the list of regions
        dict_db.update({reg:{}})

    for con in list_of_countries:  # iterate across the list of countries
        unique_main = []
        con_idx = list_of_countries.index(con)

        print('Processing: ' + con)

        eb_data = dir_balances + '/' + list_of_balances[con_idx]
        balance_file = pd.ExcelFile(eb_data)
        yearly_sheet = balance_file.sheet_names

        reg = country_2_region[con]

        dict_db[reg].update({con:{}})

        for y in range(len(yearly_sheet)):
            year = yearly_sheet[y].split(' - ')[0]

            balance_data_raw = pd.read_excel(eb_data,
                                             sheet_name=yearly_sheet[y])

            # this is custom; make sure it always applies:
            balance_data = balance_data_raw.drop([0,1,2,4,32,33,34])

            balance_data.fillna(0, inplace=True)
            balance_data.columns = balance_data.iloc[0]
            balance_data.drop([3], inplace=True)
            balance_data = balance_data.reset_index(drop=True)

            # Now we have a couple of key tasks:
            # First, extract the number of sets and find easy labels.
            # Second, nest the data appropiately.
            data_cols_all = balance_data.columns.tolist()  # copy this once
            data_cols = data_cols_all[1:]
            data_rows = balance_data[0].tolist()

            # The next thing to do is store the data nested with the logic:
            #   main row, disag row, col

            # Because of nesting logic,
            #create the dictionary vairable and sets at y == 0
            if y == 0:  # nest
                for a_main_row in list_EB_nest_main:
                    if a_main_row not in unique_main:
                        unique_main.append(a_main_row)

                        # Call main row below:
                        conv_idx = df_EB_label_name.index(a_main_row)
                        main_row_print = df_EB_labels[conv_idx]
                        dict_db[reg][con].update({main_row_print:{}})

                        # Call disaggregated row below:
                        this_index = list_EB_nest_main.index(a_main_row)
                        disag_row_raw = list_EB_nest_disag[this_index]
                        try:
                            conv_idx = df_EB_label_name.index(disag_row_raw)
                            disag_row = df_EB_labels[conv_idx]
                        except Exception:
                            disag_row = disag_row_raw

                        dict_db[reg][con][
                            main_row_print].update({disag_row:{}})

                        for col in data_cols:  # be sure to add the fuels:
                            conv_idx = df_EB_label_name.index(col)
                            col_print = df_EB_labels[conv_idx]

                            dict_db[reg][con][main_row_print][
                                disag_row].update({col_print:{}})

                    else:
                        # Call more disaggregated rows below:
                        unique_main_indices_raw = [
                            i for i in range(len(list_EB_nest_main))
                            if list_EB_nest_main[i] == a_main_row]
                        unique_main_indices = unique_main_indices_raw
                        for a_disag_idx in unique_main_indices:
                            disag_row_raw = list_EB_nest_disag[a_disag_idx]

                            try:
                                conv_idx = \
                                    df_EB_label_name.index(disag_row_raw)
                                disag_row = df_EB_labels[conv_idx]
                            except Exception:
                                disag_row = disag_row_raw

                            dict_db[reg][con][
                                main_row_print].update({disag_row:{}})

                            for col in data_cols:  # add the fuels:
                                conv_idx = df_EB_label_name.index(col)
                                col_print = df_EB_labels[conv_idx]

                                dict_db[reg][con][main_row_print][
                                    disag_row].update({col_print:{}})

            # Now, fill the data:
            list_EB_nest_main_unique = list(set(list_EB_nest_main))
            for a_main_row in list_EB_nest_main_unique:
                conv_idx = df_EB_label_name.index(a_main_row)
                main_row_print = df_EB_labels[conv_idx]

                unique_main_indices_raw = [
                    i for i in range(len(list_EB_nest_main))
                    if list_EB_nest_main[i] == a_main_row]

                unique_main_indices = unique_main_indices_raw
                for a_disag_idx in unique_main_indices:
                    disag_row_raw = list_EB_nest_disag[a_disag_idx]
                    try:
                        conv_idx = df_EB_label_name.index(disag_row_raw)
                        disag_row = df_EB_labels[conv_idx]
                    except Exception:
                        disag_row = disag_row_raw

                    for col in data_cols:
                        # Here we must extract the value:
                        if disag_row_raw != 'none':
                            mask = (balance_data[0] == disag_row_raw)
                        else:
                            mask = (balance_data[0] == a_main_row)
                        data_series = balance_data[mask]
                        if disag_row == 'Exports':
                            add_value = data_series[col].iloc[0]*-1
                        else:
                            add_value = data_series[col].iloc[0]

                        # Here we must update the dictionary
                        conv_idx = df_EB_label_name.index(col)
                        col_print = df_EB_labels[conv_idx]
                        dict_db[reg][con][main_row_print][disag_row][
                            col_print].update({year:add_value})

                        # Append the list to print in wide format:
                        wide_source.append('OLADE')
                        wide_file.append('Energy Balances')
                        wide_region.append(reg)
                        wide_country.append(con)
                        wide_variable_main.append(main_row_print)
                        wide_variable_disag.append(disag_row)
                        wide_set.append(col_print)
                        wide_year.append(year)
                        wide_value.append(add_value)

    print_output_energy_balance = True
    if print_output_energy_balance is True:
        # Store the data for visualization:
        dict_wide_database = {  'Source':deepcopy(wide_source),
                                'File':deepcopy(wide_file),
                                'Region':deepcopy(wide_region),
                                'Country':deepcopy(wide_country),
                                'Variable_Main':deepcopy(wide_variable_main),
                                'Variable_Disag':deepcopy(wide_variable_disag),
                                'Set':deepcopy(wide_set),
                                'Year':deepcopy(wide_year),
                                'Value':deepcopy(wide_value)}

        df_EB_name = 'energy_balances_wide.csv'
        df_EB = pd.DataFrame.from_dict(dict_wide_database)
        df_EB.to_csv(df_EB_name, index=None, header=True)

        # We are done with energy balances;
        # we can close everything and move to the next variable.

print('The first stage is complete.\n')
# sys.exit()

###############################################################################
if process_electrical_capacities is True:
    # 2) Second, let's process the electrical capacity databases
    print('2) Processing electrical capacity data')

    dict_db_cap = {}

    # It is also useful to have the geographic distribution df handy:
    df_geo_dist = pd.read_excel(df_mngr, sheet_name="geo_dist")

    df_cntrl_cap_label_raw = pd.read_excel(
        df_mngr, sheet_name="Cap_labeling_CON")
    df_ccl_countries = df_cntrl_cap_label_raw['countries'].tolist()

    caps_file = pd.ExcelFile(caps_filename)
    country_sheet = caps_file.sheet_names
    country_list_get = df_geo_dist['Country_Spanish_IC'].tolist()

    country_list_region = df_geo_dist['Region'].tolist()
    country_list_region_id = df_geo_dist['Reg_ID'].tolist()

    unique_reg = []
    country_list_set = []
    for c in country_sheet:
        con_spa = c.split('.')[-1]

        print('Processing: ' + con_spa)

        if con_spa in country_list_get:
            country_idx = country_list_get.index(con_spa)
            con = df_geo_dist['Country'].tolist()[country_idx]

            mask_0 = (df_therm_cap['Country'] == con)
            df_therm_cap_con = df_therm_cap.loc[mask_0]
            len_therm_cap = len(df_therm_cap_con.index.tolist())
            therm_cap_sum = df_therm_cap_con['Capacity [MW]'].sum()

            # We will now subselect the table according to the country:
            unique_con_list = list(set(df_ccl_countries))
            internal_c_idx = 0
            internal_c_idx_select = 0
            for internal_c_group in unique_con_list:
                unique_country_list = internal_c_group.split(' ; ')
                if con in unique_country_list:
                    internal_c_idx_select = deepcopy(internal_c_idx)
                internal_c_idx += 1
            country_selector = unique_con_list[internal_c_idx_select]
            country_selector_mask = \
                (df_cntrl_cap_label_raw['countries'] == country_selector)
            df_cntrl_cap_label = \
                df_cntrl_cap_label_raw.loc[country_selector_mask]

            # Call the regions below:
            reg = country_list_region[country_idx]
            reg_id = country_list_region_id[country_idx]
            regstr_prev = str(reg_id) + '_' + reg

            regstr = country_2_region[con]

            # print('check regions')
            # sys.exit()

            if regstr not in unique_reg:
                unique_reg.append(regstr)
                dict_db_cap.update({regstr:{}})

            # These lists are used below:
            df_ccl_name = df_cntrl_cap_label['name'].tolist()
            df_ccl_label = df_cntrl_cap_label['label'].tolist()
            df_ccl_fuel = df_cntrl_cap_label['fuel'].tolist()
            df_ccl_cap_share = df_cntrl_cap_label['cap_share_2019'].tolist()
            df_ccl_suffix = df_cntrl_cap_label['suffix'].tolist()

            country_list_set.append(con)
            dict_db_cap[regstr].update({con:{}})

            # Now, open the dataframe:
            df_cap_raw = pd.read_excel(caps_filename, sheet_name=c)
            df_cap = df_cap_raw.drop([0,1,2,12,13,14,15])
            df_cap.columns = df_cap.iloc[0]
            df_cap.drop([3], inplace=True)
            df_cap = df_cap.reset_index(drop=True)
            df_cap.fillna(0, inplace=True)

            old_sets = df_cap['Descripción'].tolist()
            new_sets = []
            new_sets_cap = []

            for aset in old_sets:
                setidx_list = [
                    i for i in range(len(df_ccl_name)) if
                    df_ccl_name[i] == aset]

                for setidx in setidx_list:  # This is important for modelling
                    # Naming sets:
                    grab_label = df_ccl_label[setidx]
                    grab_fuel = df_ccl_fuel[setidx]

                    if grab_label != 'as_fuel':
                        new_set = df_ccl_suffix[setidx] + '_' + grab_label + \
                            '_' + df_ccl_fuel[setidx]
                    else:
                        new_set = \
                            df_ccl_suffix[setidx] + '_' + df_ccl_fuel[setidx]
                    new_sets.append(new_set)

                    mask = (df_cap['Descripción'] == aset)
                    df_cap_grab = df_cap[mask]

                    dict_db_cap[regstr][con].update({new_set:{}})

                    for hy in hyear_vector_sh:
                        grab_cap = df_cap_grab[float(hy)].iloc[0]

                        # Here we can modify the capacity percentages:
                        if grab_label == 'Thermal':
                            grab_eq_fuels = dict_thermal_equiv[grab_fuel]
                            mask_2 = (df_therm_cap_con["Fuel"].isin(grab_eq_fuels))
                            therm_cap_fuel = df_therm_cap_con.loc[
                                mask_2]['Capacity [MW]'].sum()

                            if thermal_share_manual_mode:
                                ### print('Happen 1')
                                new_sets_cap.append(
                                    grab_cap*df_ccl_cap_share[setidx]/100)

                            if thermal_share_automatic and len_therm_cap > 0 and therm_cap_fuel >= 0:
                                ### print('Happen 2', grab_eq_fuels)
                                if therm_cap_sum == 0:
                                    print('Somebody has a zero sum of thermal', con_spa)
                                    therm_cap_sh = 0
                                    ### sys.exit()
                                else:
                                    therm_cap_sh = therm_cap_fuel/therm_cap_sum
                                new_sets_cap.append(grab_cap*therm_cap_sh)

                            else:
                                ### print('Happen 3')
                                new_sets_cap.append(
                                    grab_cap*df_ccl_cap_share[setidx]/100)

                            ### if 'Haiti' in con_spa and 'sel' in grab_eq_fuels[0]:
                            ###    print('check')
                            ###    sys.exit()

                            # print('redistribute percentages')
                            #  sys.exit()

                        else:
                            new_sets_cap.append(
                                grab_cap*df_ccl_cap_share[setidx]/100)

                        dict_db_cap[regstr][con][new_set].update(
                            {str(hy):new_sets_cap[-1]})

                        # Here we must append the list to print in wide format:
                        wide_source.append('OLADE')
                        wide_file.append('Electrical Capacity')
                        wide_region.append(regstr)
                        wide_country.append(con)
                        wide_variable_main.append('Electrical Capacity')
                        wide_variable_disag.append('Electrical Capacity')
                        wide_set.append(new_set)
                        wide_year.append(hy)
                        wide_value.append(new_sets_cap[-1])

    # We are done with electrical capacities;
    # we can close everything and move to the next variable.

print('The second stage is complete.\n')
# sys.exit()

###############################################################################
if process_electrical_capacities_raw is True:
    # 3) Third, let's process the electrical capacity databases (broadly)
    print('3) Process OLADE electrical capacity, without thermal distribution')

    # It is also useful to have the geographic distribution df handy:
    df_geo_dist = pd.read_excel(df_mngr, sheet_name="geo_dist")

    df_cntrl_cap_label = pd.read_excel(df_mngr, sheet_name="Cap_labeling")
    df_ccl_name = df_cntrl_cap_label['name'].tolist()
    df_ccl_name_eng = df_cntrl_cap_label['name_eng'].tolist()
    df_ccl_label = df_cntrl_cap_label['label'].tolist()
    df_ccl_fuel = df_cntrl_cap_label['fuel'].tolist()
    df_ccl_cap_share = df_cntrl_cap_label['cap_share_2019'].tolist()
    df_ccl_suffix = df_cntrl_cap_label['suffix'].tolist()

    caps_file = pd.ExcelFile(caps_filename)
    country_sheet = caps_file.sheet_names
    country_list_get = df_geo_dist['Country_Spanish_IC'].tolist()

    country_list_region = df_geo_dist['Region'].tolist()
    country_list_region_id = df_geo_dist['Reg_ID'].tolist()

    unique_reg = []
    country_list_set = []
    for c in country_sheet:
        con_spa = c.split('.')[-1]

        print('Processing: ' + con_spa)

        if con_spa in country_list_get:
            country_idx = country_list_get.index(con_spa)
            con = df_geo_dist['Country'].tolist()[country_idx]

            # Call the regions below:
            reg = country_list_region[country_idx]
            reg_id = country_list_region_id[country_idx]
            regstr = str(reg_id) + '_' + reg
            if regstr not in unique_reg:
                unique_reg.append(regstr)

            country_list_set.append(con)

            # Now, open the dataframe:
            df_cap_raw = pd.read_excel(caps_filename, sheet_name=c)
            df_cap = df_cap_raw.drop([0,1,2,12,13,14,15])
            df_cap.columns = df_cap.iloc[0]
            df_cap.drop([3], inplace=True)
            df_cap = df_cap.reset_index(drop=True)
            df_cap.fillna(0, inplace=True)

            old_sets = df_cap['Descripción'].tolist()
            new_sets_cap = []

            for hy in hyear_vector_sh:
                for aset in old_sets:
                    name_set = df_ccl_name_eng[df_ccl_name.index(aset)]

                    mask = (df_cap['Descripción'] == aset)
                    df_cap_grab = df_cap[mask]
                    grab_cap = df_cap_grab[float(hy)].iloc[0]

                    new_sets_cap.append(grab_cap)

                    # Here we must append the list to print in wide format:
                    wide_source.append('OLADE')
                    wide_file.append('Electrical Capacity without separation')
                    wide_region.append(regstr)
                    wide_country.append(con)
                    wide_variable_main.append('Electrical Capacity')
                    wide_variable_disag.append('Electrical Capacity')
                    wide_set.append(name_set)
                    wide_year.append(hy)
                    wide_value.append(new_sets_cap[-1])

    # We are done with electrical capacities;
    # we can close everything and move to the next variable.

print('The third stage is complete.\n')
# sys.exit()

###############################################################################
# Final) Print final dictionary:
print_final_pickle = True
if print_final_pickle is True:
    out_dict_db = {'EB': dict_db, 'Cap':dict_db_cap}
    fn_out_dict_db = 'dict_db'
    with open(fn_out_dict_db + '.pickle', 'wb') as out_dict:
        pickle.dump(out_dict_db, out_dict, protocol=pickle.HIGHEST_PROTOCOL)
    out_dict.close()

# Final) Print the final csv table:
print_output_all = True
if print_output_all is True:
    # Store the data for visualization:
    dict_wide_database = {  'Source':wide_source,
                            'File':wide_file,
                            'Region':wide_region,
                            'Country':wide_country,
                            'Variable_Main':wide_variable_main,
                            'Variable_Disag':wide_variable_disag,
                            'Set':wide_set,
                            'Year':wide_year,
                            'Value':wide_value,}

    df_all_name = 'input_database.csv'
    df_all = pd.DataFrame.from_dict(dict_wide_database)
    df_all.to_csv(df_all_name, index=None, header=True)

# Recording final time of execution:
end_f = time.time()
te_f = -start_1 + end_f  # te: time_elapsed
print(str(te_f) + ' seconds /', str(te_f/60) + ' minutes')
print('*: This automatic analysis is finished.')


import pandas as pd
import numpy as np
from queue import PriorityQueue
import json

DataStore = None

def writeLog(typ, time, data):
    print(typ, time, data)
    pass

def writeLogzambia(typ, time, data):
    pass

def readLog():
    pass

def readsub():
    pass

def readsub2():
    pass

class HierachyData():
    # filename is the file to be read
    # hiearchy is a list of hiearchy attributes in descending order
    def __init__(self, filename, hiearchy):
        if(filename.startswith("./") or filename.startswith("/")):
            self.data = pd.read_csv(filename)
        else:
            self.data = pd.read_sql("select * from " + filename, con)
        for attr in hiearchy:
            if attr not in self.data.columns:
                self.data[attr] = ""
        global DataStore
        DataStore  = self.data.copy()
        self.data = self.data.set_index(hiearchy)
        self.hiearchy = hiearchy

    def get_summary2(self):
        self.summary = self.data.reset_index()[self.hiearchy[0:4]].groupby(self.hiearchy[0:3]).agg(['unique']).to_json(orient='index')
        return self.summary

    def get_summary(self):
        self.summary = self.data.reset_index()[self.hiearchy[0:2]].groupby(self.hiearchy[0]).agg(['unique']).to_json(orient='index')
        return self.summary

    # get unique for header
    def get_unique(self,hierchy_values):
        if len(hierchy_values) == 0:
            return self.data.reset_index()[self.hiearchy[len(hierchy_values)]].unique().tolist()
        return self.data.loc[tuple(hierchy_values)].reset_index()[self.hiearchy[len(hierchy_values)]].unique().tolist()

    # get data given hierchy_values
    def get_data(self,hierchy_values):
        return self.data.loc[tuple(hierchy_values)].reset_index().to_json(orient='records')

    # get data given hierchy_values
    def get_data2(self,set_values):
        result = None

        for value in set_values:
            try:
                if(result is None):
                    result = self.data.loc[value]
                else:
                    result = result.append(self.data.loc[value])
            except:
                pass
        result["rank"] = result.groupby("District")["value"].rank(ascending=True,method='first')
        # result.groupby(categorical[0]).rank(ascending=False,method='first')
        return result.reset_index().to_json(orient='records')

    # get data given hierchy_values
    # list of two categorical attributes
    # one numerical attribute
    # and aggragation method
    def get_heatmap_data(self,hierchy_values,categorical,numerical,aggragation):

        return self.get_heatmap(hierchy_values,categorical,numerical,aggragation).to_json(orient='records')
        # return self.data.loc[tuple(hierchy_values)].groupby(categorical).agg(aggragation).reset_index().to_json(orient='records')

    def get_heatmap_withtotal(self,hierchy_values,categorical,numerical,aggragation):
        # return self.data.loc[tuple(hierchy_values)][categorical+numerical]\
        #              .groupby(categorical)[numerical].agg(aggragation).reset_index().to_json(orient='records')
        aggregation = ['mean','std','count']

        if len(hierchy_values) == 0:
            agg = self.data
        else:
            agg = self.data.loc[tuple(hierchy_values)]
        agg = agg.groupby(categorical[-1])[numerical].agg(aggregation)
        agg = agg.replace(np.nan, 0)

        agg.columns = ['mean','std','count']
        agg[categorical[0]] = 'total'
        # agg["rank"]  = (agg["mean"]*agg["count"]).rank(ascending=True)
        agg["rank"]  = (agg["mean"]*agg["count"]).rank(ascending=False,method='first')
        if len(hierchy_values) == 0:
            each = self.data
        else:
            each = self.data.loc[tuple(hierchy_values)]
        each = each.groupby(categorical)[numerical].agg(aggregation)
        each.columns = ['mean','std','count']
        each = each.replace(np.nan, 0)
        each["rank"]  = (each["mean"]*each["count"]).groupby(categorical[0]).rank(ascending=False,method='first')
        return pd.concat([agg.reset_index(),each.reset_index()])

    def get_heatmap(self,hierchy_values,categorical,numerical,aggragation):
        aggregation = ['mean','std','count']

        if len(hierchy_values) == 0:
            each = self.data
        else:
            each = self.data.loc[tuple(hierchy_values)]
        each = each.groupby(categorical)[numerical].agg(aggregation)
        each.columns = ['mean','std','count']
        each = each.replace(np.nan, 0)
        each["rank"]  = (each["mean"]*each["count"]).groupby(categorical[0]).rank(ascending=False,method='first')

        try:
            if(categorical[0] == 'sid'):
                each = each.reset_index()
                each = each.merge(self.data.reset_index()[["sid","year", "cause"]])
                return each
        except:
            pass

        return each.reset_index()

    def get_heatmap_withoutagg(self,hierchy_values,categorical,numerical,aggragation):
        # return self.data.loc[tuple(hierchy_values)][categorical+numerical]\
        #              .groupby(categorical)[numerical].agg(aggragation).reset_index().to_json(orient='records')
        aggregation = ['mean','std','count']
        each = self.data.loc[tuple(hierchy_values)].groupby(categorical).agg(aggregation)
        each.columns = ['mean','std','count']
        each["rank"]  = (each["mean"]/each["count"]).groupby(categorical[0]).rank(ascending=True)
        return each.reset_index()

    def complaint(self,hierchy_values,categorical,numerical,aggragation,year,de_mean):
        df_all = self.get_heatmap_withoutagg(hierchy_values,categorical,numerical,aggragation)
        df_all = df_all.replace(np.nan, 0)
        all_mean = df_all['mean'].median()
        all_count = df_all['count'].median()
        df = df_all[df_all.year == year]
        year_mean = df['mean'].median()
        year_count = df['count'].median()
        total_count = df['count'].sum()
        total = (df['count']*df['mean']).sum()
        cur_mean = total/total_count
        q = PriorityQueue()
        cur_dif = abs(de_mean - cur_mean)

        for i in range(df.shape[0]):

            geo = df[categorical[0]].iloc[i]

            count = df['count'].iloc[i]
            mean = df['mean'].iloc[i]

            # if there is 0 record, where mean == nan
            if(mean != mean):
                oldsum = 0
            else:
                oldsum = count*mean

            geo_data = df_all[df_all[categorical[0]]==geo].median()
            geo_mean = geo_data['mean']
            geo_count = geo_data['count']
            est_mean = geo_mean + year_mean - all_mean

            # new_mean = (total - mean*count + est_mean*count)/total_count
            # q.put((abs(de_mean - new_mean),geo, 'mean',est_mean))
            est_count = geo_count + year_count - all_count
            # new_mean = (total - mean*count + mean*est_count)/(total_count - count + est_count)
            # q.put((abs(de_mean - new_mean),geo, 'count',est_count))
            newsum = est_mean*est_count
            if(total_count - count + est_count == 0):
                new_mean = 10
            else:
                new_mean = (total - oldsum + newsum)/(total_count - count + est_count)
            if(cur_dif > abs(de_mean - new_mean)):
                q.put((abs(de_mean - new_mean),geo, est_mean ,est_count))

        result = []
        top = 4
        while not q.empty() and (top > 0):
            result.append(q.get())
            top -= 1
        return result

    # categorize an attribute
    # e.g., categorize year so that return NA for all missing years
    def categorize_attribute(self, att):

        self.data[att] = pd.Categorical(self.data[att])

def getRegionExplanation(region, year, aggOriginal, complained_agg, com_too_small):
    RegionData = DataStore.copy()
    aggs = ["count","std","mean"]

    # this gets region district mapping lost
    RegionData["year"] = pd.Categorical(RegionData["year"] )
    RegionData["region"] = pd.Categorical(RegionData["region"] )
    RegionData["district"] = pd.Categorical(RegionData["district"] )

    aggg = RegionData.groupby(["region","year","district"])["rank"].agg(["count","std","mean"])
    aggg = aggg.fillna(0)

    explanations = []

    selected = aggg.loc[(region,year)]
    pmean = selected["mean"].mean()
    pstd = selected["std"].mean()
    pcount = selected["count"].mean()

    for index, row in selected.iterrows():
        aggOld = {}
        aggNew = {}
        for agg in aggs:
            aggOld[agg] = row[agg]
        aggNew= {'mean': pmean, 'std': pstd, 'count': pcount}
        # introuce a mean 0 with count doesn't help
        if(not com_too_small and complained_agg == 'mean' and aggOld['count'] == 0 and (aggNew['count'] < 1 or aggNew['mean'] < 1 )):
            continue
        aggAfterRepair = add(remove(aggOriginal,aggOld),aggNew)
        if((com_too_small and aggAfterRepair[complained_agg] > aggOriginal[complained_agg] + 0.01) or
         (not com_too_small and aggAfterRepair[complained_agg] < aggOriginal[complained_agg] - 0.01)) :

            explanation = {}
            explanation["district"] = row.name
            explanation["year"] = year
            explanation["aggNew"] = aggNew
            explanation["after"] = aggAfterRepair[complained_agg]
            explanations.append(explanation)

    if com_too_small:
        explanations.sort(key=lambda x: -x["after"])
    else:
        explanations.sort(key=lambda x: x["after"])

    return explanations[:5]

def getDistrictExplanation(district, year, aggOriginal, complained_agg, com_too_small):
    RegionData = DataStore[DataStore['district']==district].copy()
    aggs = ["count","std","mean"]
    RegionData["year"] = pd.Categorical(RegionData["year"] )
    RegionData["district"] = pd.Categorical(RegionData["district"] )
    RegionData["village"] = pd.Categorical(RegionData["village"] )

    aggg = RegionData.groupby(["district","year","village"])["rank"].agg(["count","std","mean"])
    aggg = aggg.fillna(0)

    explanations = []

    selected = aggg.loc[(district,year)]
    pmean = selected["mean"].mean()
    pstd = selected["std"].mean()
    pcount = selected["count"].mean()

    for index, row in selected.iterrows():
        aggOld = {}
        aggNew = {}
        for agg in aggs:
            aggOld[agg] = row[agg]
        aggNew= {'mean': pmean, 'std': pstd, 'count': pcount}

        aggAfterRepair = add(remove(aggOriginal,aggOld),aggNew)

        if((com_too_small and aggAfterRepair[complained_agg] > aggOriginal[complained_agg] + 0.01) or
         (not com_too_small and aggAfterRepair[complained_agg] < aggOriginal[complained_agg] - 0.01)) :

            explanation = {}
            explanation["village"] = row.name
            explanation["year"] = year
            explanation["aggNew"] = aggNew
            explanation["after"] = aggAfterRepair[complained_agg]
            explanations.append(explanation)

    if com_too_small:
        explanations.sort(key=lambda x: -x["after"])
    else:
        explanations.sort(key=lambda x: x["after"])

    return explanations[:5]

def add(agg1, agg2):
    agg3 = {}
    agg3["count"] = agg1["count"] + agg2["count"]
    if(agg3["count"] == 0):
        agg3["mean"] = 0
        agg3["std"] = 0
        return agg3
    agg3["mean"] = (agg1["count"] * agg1["mean"] + agg2["count"] * agg2["mean"])/agg3["count"]

    if(agg3["count"] == 1):
        agg3["std"] = 0
        return agg3

    var = ((agg1["count"] - 1) * (agg1["std"]**2) + (agg2["count"] - 1) * (agg2["std"]**2)
                    + agg1["count"] * ((agg1["mean"] -  agg3["mean"]) **2)
                    + agg2["count"] * ((agg2["mean"] -  agg3["mean"]) **2) )/(agg3["count"] - 1)
    if(var < 0):
        var = 0
    agg3["std"] = var**(0.5)
    return agg3

def remove(agg1, agg2):
    agg3 = {}
    agg3["count"] = agg1["count"] - agg2["count"]

    if(agg3["count"] == 0):
        agg3["mean"] = 0
        agg3["std"] = 0
        return agg3

    agg3["mean"] = (agg1["count"] * agg1["mean"] - agg2["count"] * agg2["mean"])/agg3["count"]

    if(agg3["count"] == 1):
        agg3["std"] = 0
        return agg3

    var = ((agg1["count"] - 1) * (agg1["std"]**2) - (agg2["count"] - 1) * (agg2["std"]**2)
                    + agg1["count"] * ((agg1["mean"] -  agg3["mean"]) **2)
                    - agg2["count"] * ((agg2["mean"] -  agg3["mean"]) **2) )/(agg3["count"] - 1)
    if(var < 0):
        var = 0
    agg3["std"] = var**(0.5)
    return agg3