"""
References:
- https://github.com/facebookresearch/pytorchvideo/blob/main/pytorchvideo/data/ucf101.py
"""

from __future__ import annotations

import json
import os
import pathlib
from functools import partial
from pathlib import Path
from typing import Any, Callable

import torch
from iopath.common.file_io import g_pathmgr
from omegaconf import DictConfig
from pytorchvideo.data.clip_sampling import ClipSampler

from eztorch.datasets.decoders import DecoderType
from eztorch.datasets.labeled_video_dataset import LabeledVideoDataset
from eztorch.datasets.utils_fn import get_raw_video_duration, remove_suffix

try:
    import pandas
except ImportError:
    _HAS_PD = False
else:
    _HAS_PD = True

_UCF101_FILES = [
    "trainlist01.txt",
    "trainlist02.txt",
    "trainlist03.txt",
    "testlist01.txt",
    "testlist02.txt",
    "testlist03.txt",
]

def create_ucf101_files_for_frames(folder_files: str, frames_folder: str):
    """Create the UCF101 csv files for frame decoders.

    Args:
        folder_files: Path to the original ucf101 split files.
        frames_folder: Path to the frame folders.

    Raises:
        ImportError: If pandas is not installed.
    """

    if not _HAS_PD:
        raise ImportError("pandas is required to use this function.")

    classes = {}

    def get_video_class_index(video: str):
        video = Path(video)
        if video.parent.name not in classes:
            classes[video.parent.name] = len(classes)
        return classes[video.parent.name]

    for file in _UCF101_FILES:
        file = Path(folder_files) / file
        data = pandas.read_csv(file, sep=" ", header=None, names=["video", "label"])
        frames_folder = Path(frames_folder)

        data["label"] = data.video.map(get_video_class_index)

        data.video = data.video.map(remove_suffix)
        data["duration"] = data.video.map(
            partial(get_raw_video_duration, frames_folder)
        )
        data.to_csv(frames_folder / file.name, sep=" ", header=None, index=None)

class Ucf101LabeledVideoPaths:
    """Pre-processor for Ucf101 dataset mentioned here - https://www.crcv.ucf.edu/data/UCF101.php.

    This dataset consists of classwise folds with each class consisting of 3
        folds (splits).

    The videos directory is of the format,
        video_dir_path/class_x/<somevideo_name>.avi
        ...
        video_dir_path/class_y/<somevideo_name>.avi

    The splits/fold directory is of the format,
        folds_dir_path/classInd.txt
        folds_dir_path/testlist01.txt
        folds_dir_path/testlist02.txt
        folds_dir_path/testlist03.txt
        folds_dir_path/trainlist01.txt
        folds_dir_path/trainlist02.txt
        folds_dir_path/trainlist03.txt

    Args:
        paths_and_labels: a list of tuples containing the video
            path and integer label.
    """

    _allowed_splits = [1, 2, 3]

    @classmethod
    def from_path(
        cls,
        data_path: str,
        split_id: int = 1,
        split_type: str = "train",
        frames: bool = False,
    ) -> Ucf101LabeledVideoPaths:
        """Factory function that creates a LabeledVideoPaths object depending on the path type.

        - If it is a directory path it uses the LabeledVideoPaths.from_directory function.
        - If it's a file it uses the LabeledVideoPaths.from_csv file.
        Args:
            file_path: The path to the file or directory to be read.
            split_id: Split id. Used if path is a directory.
            split_type: Split type. Used if path is a directory.
            frames: If ``True``, UCF101 is loaded as a frame dataset.
        Returns:
            The Ucf101LabeledVideoPaths object.
        """

        if g_pathmgr.isfile(data_path):
            if pathlib.Path(data_path).suffix == ".json":
                return Ucf101LabeledVideoPaths.from_json(data_path)
            return Ucf101LabeledVideoPaths.from_csv(data_path)
        elif g_pathmgr.isdir(data_path):
            return Ucf101LabeledVideoPaths.from_directory(
                data_path, split_id, split_type, frames
            )
        else:
            raise FileNotFoundError(f"{data_path} not found.")

    @classmethod
    def from_csv(cls, file_path: str) -> Ucf101LabeledVideoPaths:
        """Factory function that creates a LabeledVideoPaths object by reading a file with the following format:

            <path> <integer_label>
            ...
            <path> <integer_label>

        Args:
            file_path: The path to the file to be read.
        Returns:
            The Ucf101LabeledVideoPaths object.
        """
        assert g_pathmgr.exists(file_path), f"{file_path} not found."
        video_paths_and_label = []
        with g_pathmgr.open(file_path, "r") as f:
            for path_label in f.read().splitlines():
                line_split = path_label.rsplit(None, 2)

                # The video path file may not contain labels (e.g. for a test split). We
                # assume this is the case if only 1 path is found and set the label to
                # -1 if so.
                if len(line_split) == 1:
                    file_path = line_split[0]
                    label = -1
                    num_frames = None
                elif len(line_split) == 2:
                    file_path, label = line_split
                    num_frames = None
                else:
                    file_path, label, num_frames = line_split
                    num_frames = int(num_frames)
                video_paths_and_label.append((file_path, int(label), num_frames))

        assert (
            len(video_paths_and_label) > 0
        ), f"Failed to load dataset from {file_path}."
        return cls(video_paths_and_label)

    @classmethod
    def from_json(cls, file_path: str) -> Ucf101LabeledVideoPaths:
        """Factory function that creates a LabeledVideoPaths object by reading a json file.

        Args:
            file_path: The path to the file to be read.
        Returns:
            The Ucf101LabeledVideoPaths object.
        """
        assert g_pathmgr.exists(file_path), f"{file_path} not found."
        video_paths_and_label = []

        json_content = json.load(open(file_path))

        annotation = json_content["annotation"]
        videos_id = sorted([x for x in annotation.keys()])

        for video_id in videos_id:
            label = annotation[video_id]["class"]
            num_frames = (
                annotation[video_id]["num_frames"]
                if "num_frames" in annotation["video_id"]
                else None
            )
            video_path = f"{video_id}"
            video_paths_and_label.append((video_path, int(label), int(num_frames)))

        assert (
            len(video_paths_and_label) > 0
        ), f"Failed to load dataset from {file_path}."
        return cls(video_paths_and_label)

    @classmethod
    def from_directory(
        cls,
        data_path: str,
        split_id: int = 1,
        split_type: str = "train",
        frames: bool = False,
    ) -> Ucf101LabeledVideoPaths:
        """Factory function that creates Ucf101LabeledVideoPaths object form a splits/folds directory.

        Args:
            data_path: The path to the splits/folds directory of UCF-101.
            split_id: Fold id to be loaded. Options are: :math:`1`, :math:`2`, :math:`3`.
            split_type: Split/Fold type to be loaded. Options are: ``'train'``, ``'test'``.
            frames: If ``True``, UCF101 is loaded as a frame dataset.

        Returns:
            The Ucf101LabeledVideoPaths object.
        """
        data_path = pathlib.Path(data_path)
        if not data_path.is_dir():
            raise RuntimeError(f"{data_path} not found or is not a directory.")
        if not int(split_id) in cls._allowed_splits:
            raise RuntimeError(
                f"{split_id} not found in allowed split id's {cls._allowed_splits}."
            )
        file_name = data_path / f"{split_type}list0{split_id}.txt"
        label_file_name = data_path / "classInd.txt"
        return cls.from_csvs(file_name, label_file_name, frames)

    @classmethod
    def from_csvs(
        cls,
        file_path: pathlib.Path | str,
        label_file_name: pathlib.Path | str,
        frames: bool = False,
    ) -> Ucf101LabeledVideoPaths:
        """Factory function that creates Ucf101LabeledVideoPaths object form a list of split files of .txt type.

        Args:
            file_paths : The path to the splits/folds
                    directory of UCF-101.
            split_type: Split/Fold type to be loaded.
                - "train"
                - "test"

        Returns:
            The Ucf101LabeledVideoPaths object.
        """
        label_file_path = pathlib.Path(label_file_name)
        assert g_pathmgr.exists(label_file_path), f"{label_file_path} not found."
        if not (
            label_file_path.suffix == ".txt" and label_file_path.stem == "classInd"
        ):
            raise RuntimeError(f"Invalid file: {file_path}")
        class_dict = {}
        with g_pathmgr.open(label_file_path, "r") as f:
            for line in f.read().splitlines():
                line_split = line.rsplit(None, 1)
                class_dict[line_split[1]] = int(line_split[0]) - 1

        video_paths_and_label = []
        file_path = pathlib.Path(file_path)
        assert g_pathmgr.exists(file_path), f"{file_path} not found."
        if not (file_path.suffix == ".txt"):
            raise RuntimeError(f"Invalid file: {file_path}")
        with g_pathmgr.open(file_path, "r") as f:
            for line in f.read().splitlines():
                line_split = line.rsplit(None, 2)

                video_name = pathlib.Path(line_split[0])
                if frames:
                    video_path = (
                        pathlib.Path(file_path.parent)
                        / video_name.parent
                        / video_name.stem
                    )
                    num_frames = len(list(video_path.iterdir()))
                else:
                    video_path = pathlib.Path(file_path.parent) / video_name
                    num_frames = None
                video_label = class_dict[str(video_path.parent.stem)]

                video_paths_and_label.append((video_path, video_label, num_frames))

        assert (
            len(video_paths_and_label) > 0
        ), f"Failed to load dataset from {file_path}."
        return cls(video_paths_and_label)

    def __init__(
        self, paths_and_labels: list[tuple[str, dict | None]], path_prefix=""
    ) -> None:
        self._paths_and_labels = paths_and_labels
        self._path_prefix = path_prefix

    def path_prefix(self, prefix):
        self._path_prefix = prefix

    path_prefix = property(None, path_prefix)

    def __getitem__(self, index: int) -> tuple[str, dict]:
        """
        Args:
            index: the path and label index.

        Returns:
            The path and label tuple for the given index.
        """
        path, label, num_frames = self._paths_and_labels[index]
        return (
            os.path.join(self._path_prefix, path),
            {"label": label, "num_frames": num_frames},
        )

    def __len__(self) -> int:
        """
        Returns:
            The number of video paths and label pairs.
        """
        return len(self._paths_and_labels)

def Ucf101(
    data_path: str,
    clip_sampler: ClipSampler,
    transform: Callable[[dict[str, Any]], dict[str, Any]] | None = None,
    video_path_prefix: str = "",
    split_id: int = 1,
    split_type: str = "train",
    decode_audio: bool = True,
    decoder: str = "pyav",
    decoder_args: DictConfig = {},
) -> LabeledVideoDataset:
    """A helper function to create ``LabeledVideoDataset`` object for the Ucf101 dataset.

    Args:
        data_path: Path to the data. The path type defines how the data
            should be read:

            * For a file path, the file is read and each line is parsed into a
              video path and label.
            * For a directory, the directory structure defines the classes
              (i.e. each subdirectory is a class).

        clip_sampler: Defines how clips should be sampled from each
                video. See the clip sampling documentation for more information.

        video_sampler: Sampler for the internal
                video container. This defines the order videos are decoded and,
                if necessary, the distributed split.

        transform: This callable is evaluated on the clip output before
                the clip is returned. It can be used for user defined preprocessing and
                augmentations to the clips. See the ``LabeledVideoDataset`` class for clip
                output format.

        video_path_prefix: Path to root directory with the videos that are
                loaded in ``LabeledVideoDataset``. All the video paths before loading
                are prefixed with this path.

        split_id: Fold id to be loaded. Options are: :math:`1`, :math:`2` or :math:`3`.

        split_type: Split/Fold type to be loaded. Options are: ``'train'`` or ``'test'``.

        decode_audio: If ``True``, also decode audio from video.

        decoder: Defines what type of decoder used to decode a video.

        decoder_args: Arguments to configure the decoder.

    Returns:
            The dataset instantiated.
    """

    torch._C._log_api_usage_once("PYTORCHVIDEO.dataset.Ucf101")
    labeled_video_paths = Ucf101LabeledVideoPaths.from_path(
        data_path,
        split_id=split_id,
        split_type=split_type,
        frames=DecoderType(decoder) == DecoderType.FRAME,
    )

    labeled_video_paths.path_prefix = video_path_prefix
    dataset = LabeledVideoDataset(
        labeled_video_paths,
        clip_sampler,
        transform,
        decode_audio,
        decoder,
        decoder_args,
    )

    return dataset


import logging

import oauth
import settings
import utils
import webapp2

URL = 'https://api.twitter.com'

OAUTH_APP_SETTINGS = {
	'consumer_key': settings.TWITTER_KEY,
	'consumer_secret': settings.TWITTER_SECRET,
	'request_token_url': URL + '/oauth/request_token',
	'access_token_url': URL + '/oauth/access_token',
	'user_auth_url': URL + '/oauth/authorize',
	'default_api_prefix': URL,
	'default_api_suffix': '.json',
	'oauth_callback': None, # set later, after webapp2 is configured
}

def oauth_client(app, *args):
	if not OAUTH_APP_SETTINGS['oauth_callback']:
		OAUTH_APP_SETTINGS['oauth_callback'] = utils.absolute_uri('twitter', action='callback')
	return oauth.OAuthClient(app, OAUTH_APP_SETTINGS, *args)

def oauth_token(*args):
	return oauth.OAuthToken(*args)


from numpy.core.numeric import isclose
import pandas as pd
from pandas import ExcelWriter
import numpy as np
import math
import random
from pandas.core.arrays.integer import Int32Dtype
from pandas.core.indexes.api import get_objs_combined_axis

SCHOOL_FILE= 'Grade Level_20210509_Names_Removed.xls'

# The file provided by the school indicates students' previous grade level.
EXIT_GRADES= ['KG', '01', '02', '03', '04', '05']

# These are the "yes/no or male/female" categories
BINARY_CATEGORIES=['504 - 2020-2021','Gender','LAP Indicator - 2020-2021']

# Special Education and gifted student require clustering.
CLUSTERED_CATEGORIES =['SPED','HCP - 2020-2021']

# This category group is based on numerical standardized test scores.
NOMINAL_CATEGORIES = ['IRLA-Score - 2020-2021', 'iReady-ELA Score Winter - 2020-2021', 'iReady-Math Score Winter - 2020-2021',]

# Attendance metric divides into 3 groups per client requirements.
ATTENDANCE_CATEGORY = ['Attn % - 2020-2021']

# Race is a special category because affinity clusters' and balance across classrooms will be solved.
SPECIAL_CATEGORY = ['Race']

#These are the only races defined by School Information System Database.
RACES = ['Asian', 'Black', 'Hispanic', 'Native', 'Multiple', 'Pacific Islander', 'White']

ALL_CATEGORIES = [BINARY_CATEGORIES + CLUSTERED_CATEGORIES + NOMINAL_CATEGORIES + ATTENDANCE_CATEGORY + RACES]

# This file defines acceptable clusters for special education based on staffing resources.
CLUSTER_FILE = 'SPED_Clusters.xlsx'

# This file defines acceptable clusters for race affinity groups.
CLUSTER_FILE_FLOAT = 'Race_Affinity_Clusters.xlsx'

# Tolerances set by end user, but established in this instance by collaborating school administrator (client).  Increasing these values increases runtime, but increases balance.
GENDER_TOL = .20
IEP_TOL = 1
LAP_TOL = .25
ATTENDANCE_TOL = .3
IRLA_TOL = .25

def convert_attribs(student_body):
    """Clean white space from student body file.  Convert 'yes/no' or 'male/female' categories to 1's and 0's.  Convert attendance category from string type % to float type %."""

    # Trim all whitespace from dataframe.
    student_body.applymap(lambda x: x.strip() if isinstance(x, str) else x)

    # Change 'yes/no' categories to '1' or '0'.
    student_body['SPED'] = np.where(student_body['SPED'] == 'Yes', 1, 0)
    student_body['HCP - 2020-2021'] = np.where(student_body['HCP - 2020-2021'].str.contains('Yes'), 1, 0)
    student_body['504 - 2020-2021'] = np.where(student_body['504 - 2020-2021'].str.contains('Yes'), 1, 0)
    student_body['Gender'] = np.where(student_body['Gender'].str.contains('Male'), 1, 0)
    student_body['LAP Indicator - 2020-2021'] = np.where(student_body['LAP Indicator - 2020-2021'].str.contains('Yes'), 1, 0)

    # Convert string attendance % to float.
    student_body['Attn % - 2020-2021'] = student_body['Attn % - 2020-2021'].str.replace('%', '').astype(float)

    student_body = categorize_attendance(student_body)

    return student_body

def classes_per_grade(grade, num):
    """Ask administrator how many classes should be formed based
    on the number of students."""

    # Transition students to the following year's grade.  File provided as input to this program indicates students' previous year's grade level
    grade +=1

    # Ask for input from user based on staff resources available next year.
    print(f"There are {num} students coming into grade {grade} next year.")
    classes = float(input("\nHow many classes for this grade would you like to form?"))

    return classes

def num_per_grade(all_grades):
    """Count the number of students in each grade and return a list containing the number of students per exiting grade. Index 0= Kindergarten, 1= First Grade, etc."""

    sizes = []
    for students in all_grades:
        students_per_grade = (len(students.index))
        sizes.append(students_per_grade)

    return sizes

def initialize_data():
    """Import student data from file and create a dataframe, call function to convert string data to ints and floats, and group students by grade. """

    # Import student data from school district program output.
    all_students = pd.read_excel(SCHOOL_FILE, index_col = [1], header=[10])

    # Convert string categories to integers and floats.
    conv_students = convert_attribs(all_students)

    # Group students by grade.
    gradegroups = conv_students.groupby(conv_students.Grade)
    kg = gradegroups.get_group('KG')
    firstg = gradegroups.get_group('01')
    secondg = gradegroups.get_group('02')
    thirdg = gradegroups.get_group('03')
    fourthg = gradegroups.get_group('04')
    fifthg = gradegroups.get_group('05')

    students_by_grade = [kg, firstg, secondg, thirdg, fourthg, fifthg]

    return students_by_grade

def how_many_classes(st_by_grade):
    """Determine how many classes will be formed and return a list which indicates number of students per grade."""

    # Find the amount of students in each grade
    grade_sizes = num_per_grade(st_by_grade)

    # Ask school administrator how many classes per grade based on staffing
    num_classes = []

    for this_grade in range(len(grade_sizes)):
        classes = classes_per_grade(this_grade, grade_sizes[this_grade])
        num_classes.append(classes)

    return num_classes

def shuffle(grades_to_shuffle):
    """Shuffle students randomly and return a list of dataframes of students grouped by grade."""

    shuffled = []
    for grades in grades_to_shuffle:
        random = grades.sample(frac=1)
        shuffled.append(random)
    return shuffled

def divide_into_classes(grades, n_classes):
    """Split all students in the same grade by the number of classes which was provided by the school administrator."""

    next_grades = []
    i = 0
    for x in grades:
        num = n_classes[i]
        nextgrade = np.array_split(x, num)
        next_grades.append(nextgrade)
        i+=1
    return next_grades

def missing_scores_school(all_classes_by_grade):
    """Fills missing test scores of each grade in the whole student body with the average test score of that grade and returns the whole student body file cleaned.  Only conducted once."""

    ''' TODO ask client whether or not empty scores should use the average of the other test scores or use a lower score due to the fact that it looks like those who have missing scores are underperforming in other areas as well'''

    for grade in all_classes_by_grade:
        for cls in grade:
            cls = cls.fillna(cls.mean())

    return all_classes_by_grade

def reshuffle_one_grade(grade):
    """Reshuffle one grade of students and return the grade student body."""

    num = len(grade)
    reshuffled = grade.loc[random.sample(list(grade.index),num)]

    return reshuffled

def divide_one_grade(grade, n_classes):
    """Divide one grade of students by the number of classes and return the classes of students."""

    classes= np.array_split(grade, n_classes)
    return classes

def missing_scores_gradelevel(grade):
    """Fill missing test scores of each class in one grade with the average of the one class and return the dataframe of the grade."""

    ''' TODO ask client wether or not empty scores should use the average of the other test scores, or use a lower score due to the fact that it looks like those who have missing scores are underperforming in other areas as well.'''

    for cls in grade:
        cls = cls.fillna(cls.mean())

    return grade

def categorize_attendance(all_students):
    """Convert student body attendance from % to three categories."""

    all_students.loc[all_students['Attn % - 2020-2021'] > 90, 'Attn % - 2020-2021'] = 0
    all_students.loc[all_students['Attn % - 2020-2021'] > 80, 'Attn % - 2020-2021'] = 1
    all_students.loc[all_students['Attn % - 2020-2021'] > 3, 'Attn % - 2020-2021'] = 2
    return all_students

def calculate(classes_to_check):
    """For every class in every grade, calc averages and other data, and store them in a list of new dataframes and return the list. This only runs once."""

    gender_list = []
    iep_list = []
    lap_list = []
    sped_list = []
    hcp_list = []
    grade_list = []
    att_list = []
    irla_list = []
    asian_list = []
    black_list = []
    hispanic_list =[]
    multi_list = []
    native_list = []
    pi_list = []
    white_list = []

    racedf = pd.DataFrame(columns = RACES)

    for grade_levels in classes_to_check:
        i =0
        for c in grade_levels:
            iep_calc = c['504 - 2020-2021'].sum()/len(c)
            iep_list.append(iep_calc)
            att_calc = c['Attn % - 2020-2021'].sum()/len(c)
            att_list.append(att_calc)
            gender_calc = c['Gender'].sum()/len(c)
            gender_list.append(gender_calc)
            lap_calc = c['LAP Indicator - 2020-2021'].sum()/len(c)
            lap_list.append(lap_calc)
            sped_total = c['SPED'].sum()
            sped_list.append(sped_total)
            hcp_total = c['HCP - 2020-2021'].sum()
            hcp_list.append(hcp_total)
            irla_calc = c['IRLA-Score - 2020-2021'].sum()/len(c)
            irla_list.append(irla_calc)
            race_int= c['Race'].value_counts()
            racedf = racedf.append(race_int, ignore_index=True)
            racedf = racedf.fillna(0)
            asian_list = racedf['Asian'].values.tolist()
            black_list = racedf['Black'].to_list()
            hispanic_list = racedf['Hispanic'].to_list()
            multi_list = racedf['Multiple'].to_list()
            native_list = racedf['Native'].to_list()
            pi_list = racedf['Pacific Islander'].to_list()
            white_list = racedf['White'].to_list()
            g = c.iloc[0]['Grade']
            grade_list.append(g)

        i= i +1

    # Instantiate and populate a df for calculations.
    calc = pd.DataFrame(columns=ALL_CATEGORIES,index=grade_list).rename_axis('Classes')
    calc['Gender']= gender_list
    calc['504 - 2020-2021'] = iep_list
    calc['LAP Indicator - 2020-2021'] = lap_list
    calc['SPED'] = sped_list
    calc['HCP - 2020-2021'] = hcp_list
    calc['Attn % - 2020-2021'] = att_list
    calc['IRLA-Score - 2020-2021'] = irla_list
    calc['Asian'] = asian_list
    calc['Black'] = black_list
    calc['Hispanic'] = hispanic_list
    calc['Multiple'] = multi_list
    calc['Native'] = native_list
    calc['Pacific Islander'] = pi_list
    calc['White'] = white_list

    # Create new dfs for each grade's calculations.
    calc1 = calc.loc[calc.index == EXIT_GRADES[0]]
    calc2 = calc.loc[calc.index == EXIT_GRADES[1]]
    calc3 = calc.loc[calc.index == EXIT_GRADES[2]]
    calc4 = calc.loc[calc.index == EXIT_GRADES[3]]
    calc5 = calc.loc[calc.index == EXIT_GRADES[4]]
    calc6 = calc.loc[calc.index == EXIT_GRADES[5]]
    calc_dfs = [calc1, calc2, calc3, calc4, calc5, calc6]

    return calc_dfs

def check_clusters(grade_to_check):
    """Determine if the special education and gifted student of this grade fall into acceptable clusters defined by school."""

    nclasses = len(grade_to_check)
    clusters_good = True
    sped_clus_good = True
    hicap_clus_good = True
    school_clusters = pd.read_excel(CLUSTER_FILE, header=None, sheet_name=((nclasses)-2))
    cluster_lists = school_clusters.values.tolist()

    # If there's only one class, clusters are not applicable.
    if nclasses == 1:
        return clusters_good
        pass
    else:
        sped = sorted(grade_to_check['SPED'].values.tolist())
        sped = [i[0] for i in sped]
        hcp = sorted(grade_to_check['HCP - 2020-2021'].values.tolist())
        hcp = [i[0] for i in hcp]
        if sped in cluster_lists:
            sped_clus_good = True
        else:
            sped_clus_good = False
        if hcp in cluster_lists:
            hicap_clus_good = True
        else:
            hicap_clus_good = False
        clusters_good = hicap_clus_good and sped_clus_good
        return clusters_good

def affinity_diversity_check(grade_to_check):
    """Determine if the special education and gifted student of this grade fall into acceptable clusters defined by school and return true or false."""

    nclasses = len(grade_to_check)
    check_good = True
    school_clusters = pd.read_excel(CLUSTER_FILE_FLOAT, header=None, sheet_name=((nclasses)-2), dtype=str)
    school_clusters = school_clusters.astype(float)
    cluster_lists = school_clusters.values.tolist()
    race_dict = {}

    # If there's only one class, clusters are not applicable.
    if nclasses == 1:
        check_good = True
        return check_good
        pass
    else:
        i = 0
        for race in RACES:
            race_list = sorted(grade_to_check[race].values.tolist())
            race_list = [i[0] for i in race_list]
            race_sum = sum(race_list)
            #If there are 10 or more of one race we will not check for clusters. The rationale being that affinity at school is somewhat less important for large race groups. TODO This could lead to students without affinity (one student of a given race in a classroom with no race peers). This is not ideal no matter the student's race. Solving for a larger number of one race has led to extensive runtimes.
            if (race_sum > 9):
                race_dict[i] = True
            else:
                if (sorted(race_list) in cluster_lists):
                    race_dict[i] = True
                else:
                    race_dict[i] = False
            i = i +1

    check_good = all(race_dict.values())
    return check_good

def calculate_one_grade(grade):
    """For every class in the one grade provided, calc averages and other data, and store them in a new dataframe and return the dataframe."""

    """TODO This function is almost identical to the 'calculate' function, except that this code runs when the first classrooms set by 'calculate' do not meet acceptance tolerances. Restructure code in order to be more elegant and pythonic. Ran out of time."""

    gender_list = []
    iep_list = []
    lap_list = []
    sped_list = []
    hcp_list = []
    grade_list = []
    att_list = []
    irla_list = []
    asian_list = []
    black_list = []
    hispanic_list =[]
    multi_list = []
    native_list = []
    pi_list = []
    white_list = []

    racedf = pd.DataFrame(columns = RACES)

    for c in grade:
        iep_calc = c['504 - 2020-2021'].sum()/len(c)
        iep_list.append(iep_calc)
        att_calc = c['Attn % - 2020-2021'].sum()/len(c)
        att_list.append(att_calc)
        gender_calc = c['Gender'].sum()/len(c)
        gender_list.append(gender_calc)
        lap_calc = c['LAP Indicator - 2020-2021'].sum()/len(c)
        lap_list.append(lap_calc)
        sped_total = c['SPED'].sum()
        sped_list.append(sped_total)
        hcp_total = c['HCP - 2020-2021'].sum()
        hcp_list.append(hcp_total)
        irla_calc = c['IRLA-Score - 2020-2021'].sum()/len(c)
        irla_list.append(irla_calc)
        race_int= c['Race'].value_counts()
        racedf = racedf.append(race_int, ignore_index=True)
        racedf = racedf.fillna(0)
        asian_list = racedf['Asian'].values.tolist()
        black_list = racedf['Black'].to_list()
        hispanic_list = racedf['Hispanic'].to_list()
        multi_list = racedf['Multiple'].to_list()
        native_list = racedf['Native'].to_list()
        pi_list = racedf['Pacific Islander'].to_list()
        white_list = racedf['White'].to_list()
        g = c.iloc[0]['Grade']
        grade_list.append(g)

    new_calc_df = pd.DataFrame(columns=ALL_CATEGORIES,index=grade_list).rename_axis('Classes')
    new_calc_df['Gender']= gender_list
    new_calc_df['504 - 2020-2021'] = iep_list
    new_calc_df['LAP Indicator - 2020-2021'] = lap_list
    new_calc_df['SPED'] = sped_list
    new_calc_df['HCP - 2020-2021'] = hcp_list
    new_calc_df['Attn % - 2020-2021'] = att_list
    new_calc_df['IRLA-Score - 2020-2021'] = irla_list
    new_calc_df['Asian'] = asian_list
    new_calc_df['Black'] = black_list
    new_calc_df['Hispanic'] = hispanic_list
    new_calc_df['Multiple'] = multi_list
    new_calc_df['Native'] = native_list
    new_calc_df['Pacific Islander'] = pi_list
    new_calc_df['White'] = white_list

    return new_calc_df

def save_xlsx(list_dfs, xlsx_path):
    """Write acceptable classes to Excel file."""

    with ExcelWriter(xlsx_path) as writer:
        i=0
        for x in list_dfs:
            for n, df in enumerate(x):
                df.to_excel(writer,'sheet%s' % i)
                i = i+1
        writer.save()

def main():
    """This program accepts as input a file from a elementary school district's Student Identification System(SIS), randomly assigns student to classes for the next year, performs balance and cluster checking in accordance with user settings, then repeats until all balance and clustering are within tolerance. The program then writes next year's class lists to an Excel file for use by the school. For this prototype, a specific school collaborated with this project, and that school's district uses SIS software distributed by Synergy."""

    # Shuffle students within their grade levels.
    shuffled_stu_by_grade = shuffle(students_by_grade)

    # Divide students into classes.
    all_classes_by_grade = divide_into_classes(shuffled_stu_by_grade, cls_per_grade)

    # Fill in missing test scores with average for the class
    cleaned_grades = missing_scores_school(all_classes_by_grade)

    # Create dataframes containing calculations for each class
    calc_dfs = calculate(cleaned_grades)

    next_yrs_classes = []

    i=0

    # For each grade level, find the class that has the most and least of all binary categories.
    for g in cleaned_grades:
        print('Attempting to solve the outgoing', str(g[0].iloc[0]['Grade']), 'grade...')
        gender_max = calc_dfs[i]['Gender'].max()
        gender_min = calc_dfs[i]['Gender'].min()
        iep_max = calc_dfs[i]['504 - 2020-2021'].max()
        iep_min = calc_dfs[i]['504 - 2020-2021'].min()
        lap_max = calc_dfs[i]['LAP Indicator - 2020-2021'].max()
        lap_min = calc_dfs[i]['LAP Indicator - 2020-2021'].min()
        att_max = calc_dfs[i]['Attn % - 2020-2021'].max()
        att_min = calc_dfs[i]['Attn % - 2020-2021'].min()
        irla_max = calc_dfs[i]['IRLA-Score - 2020-2021'].max()
        irla_min = calc_dfs[i]['IRLA-Score - 2020-2021'].min()

        # Check special education and gifted clusters
        cluster_check = check_clusters(calc_dfs[i])

        # Check for appropriate race affinity clusters
        affinity_diversity = affinity_diversity_check(calc_dfs[i])

        # If all tolerances and clusters are acceptable, save this class to the class list.
        if((math.isclose(gender_max, gender_min, abs_tol = GENDER_TOL)) and (math.isclose(iep_max, iep_min, abs_tol = IEP_TOL)) and (math.isclose(lap_max, lap_min, abs_tol = LAP_TOL)) and (math.isclose(att_max, att_min, abs_tol = ATTENDANCE_TOL)) and (math.isclose(irla_max, irla_min, abs_tol = ATTENDANCE_TOL)) and cluster_check and affinity_diversity):
            next_yrs_classes.append(g)

        # If one tolerance is out of specification, or one category does not have acceptable clustering, repeat shuffling and checking.
        else:
            while ((math.isclose(gender_max, gender_min, abs_tol = GENDER_TOL)) and (math.isclose(iep_max, iep_min, abs_tol = IEP_TOL)) and (math.isclose(lap_max, lap_min, abs_tol = LAP_TOL)) and (math.isclose(att_max, att_min, abs_tol = ATTENDANCE_TOL)) and  (math.isclose(irla_max, irla_min, abs_tol = ATTENDANCE_TOL)) and cluster_check and affinity_diversity) == False:

                r = reshuffle_one_grade(shuffled_stu_by_grade[i])
                new_classes = divide_one_grade(r, cls_per_grade[i])
                cleaned_grade = missing_scores_gradelevel(new_classes)
                new_calc_df = calculate_one_grade(cleaned_grade)

                print('Attempting to solve the outgoing', str(g[0].iloc[0]['Grade']), 'grade for next year...')
                print('Gender balance within tolerance?                                             ', str(math.isclose(gender_max, gender_min, abs_tol = GENDER_TOL)), '                    ', str(g[0].iloc[0]['Grade']))
                print('IEP student balance within tolerance?                                        ', str(math.isclose(iep_max, iep_min, abs_tol = IEP_TOL)), '                    ', str(g[0].iloc[0]['Grade']))
                print('Students with LAP indicator balance within tolerance?                        ', str(math.isclose(lap_max, lap_min, abs_tol = LAP_TOL)), '                    ', str(g[0].iloc[0]['Grade']))
                print('Historical attendance balance within tolerance?                              ', str(math.isclose(att_max, att_min, abs_tol = ATTENDANCE_TOL)), '                    ', str(g[0].iloc[0]['Grade']))
                print('Each class balanced for IRLA scores?                                         ', str(math.isclose(irla_max, irla_min, abs_tol = ATTENDANCE_TOL)), '                    ', str(g[0].iloc[0]['Grade']))
                print('Each class checked for acceptable special education and HiCap clusters?      ', str(cluster_check), '                    ', str(g[0].iloc[0]['Grade']))
                print('Race affinity and diversity checks complete?                                 ', str(affinity_diversity), '                    ', str(g[0].iloc[0]['Grade']))

                gender_max = new_calc_df['Gender'].max()
                gender_min = new_calc_df['Gender'].min()
                iep_max = new_calc_df['504 - 2020-2021'].max()
                iep_min = new_calc_df['504 - 2020-2021'].min()
                lap_max = new_calc_df['LAP Indicator - 2020-2021'].max()
                lap_min = new_calc_df['LAP Indicator - 2020-2021'].min()
                att_max = new_calc_df['Attn % - 2020-2021'].max()
                att_min = new_calc_df['Attn % - 2020-2021'].min()
                irla_max = new_calc_df['IRLA-Score - 2020-2021'].max()
                irla_min = new_calc_df['IRLA-Score - 2020-2021'].min()
                cluster_check = check_clusters(new_calc_df)
                affinity_diversity = affinity_diversity_check(new_calc_df)

            """TODO unclean final classes by unfilling missing test scores so school will have correct data (missing test scores) in their new class lists"""

            next_yrs_classes.append(new_classes)
        i = i+1

    s = ("Congratulations! Your classes have been formed with the following tolerances: \n"
            "Every class within each grade level has within " + (str(GENDER_TOL*100)) + "% the same number of boys.\n"
            "Every class within each grade level has within " + (str(IEP_TOL*100)) + "% the same number of students who have an IEP. \n"
            "Every class within each grade level has within " + (str(LAP_TOL*100)) + "% the same number of students who have a Learning Acquisition Plan indicator. \n"
            "After grouping attendance into three categories (Above 90%, 90-80%, and below 80%), every class within each grade level has a distribution of students in these categories within " + (str(ATTENDANCE_TOL*100)) + "%. \n"
            "Every class within each grade level has balanced achievement. The average of students' standardized test scores for each class is within " + (str(IRLA_TOL*100)) + "%. \n"
            "All special education and highly capable students are clustered in accordance with your cluster file, and students' races are balanced across classrooms with special affinity groups assigned in accordance with your cluster file. \n")

    print(s)
    save_xlsx(next_yrs_classes, r'C:\Users\derek\Documents\CityU\CS687_Capstone\Classroom Equity Folder\NextYrsClasses.xlsx' )

# Clean data and divide students by grade
students_by_grade = initialize_data()

# Determine how many classes each grade level will form for next year.
cls_per_grade = how_many_classes(students_by_grade)

main()


'''
Activity: make the ball bounce off the window borders.
'''

import pygame
pygame.init()

# Set up the drawing window
WINDOW_WIDTH, WINDOW_HEIGHT = 500, 500
screen = pygame.display.set_mode([WINDOW_WIDTH, WINDOW_HEIGHT])

running = True

CIRCLE_COLOR = (0, 0, 255)
CIRCLE_RADIUS = 50
CIRCLE_X = 250
CIRCLE_Y = 250

X_VELOCITY = 0.02
Y_VELOCITY = -0.02

while running:
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            running = False

    screen.fill((255, 255, 255))
    pygame.draw.circle(screen, CIRCLE_COLOR, (CIRCLE_X, CIRCLE_Y), CIRCLE_RADIUS)
    pygame.display.flip()

    '''
    Update the ball position !
    '''
    CIRCLE_X += X_VELOCITY
    CIRCLE_Y += Y_VELOCITY

pygame.quit()


import torch.nn as nn
from argparse import ArgumentParser

def conv3x1(in_planes, out_planes, stride=1, is_transpose=False):
    """3x1 convolution with padding"""
    if is_transpose:
            return nn.ConvTranspose2d(in_planes, out_planes, kernel_size=(3,1), stride=stride,
                     padding=(1,0), bias=False)
    return nn.Conv2d(in_planes, out_planes, kernel_size=(3,1), stride=stride,
                     padding=(1,0), bias=False)

class BasicBlock(nn.Module):
    expansion = 1

    def __init__(self, inplanes, planes, stride=1, downsample=None, is_transpose=False):
        super(BasicBlock, self).__init__()
        self.conv1 = conv3x1(inplanes, planes, stride, is_transpose=is_transpose)
        self.bn1 = nn.BatchNorm2d(planes)
        self.relu = nn.ReLU(inplace=True)
        self.conv2 = conv3x1(planes, planes, is_transpose=is_transpose)
        self.bn2 = nn.BatchNorm2d(planes)
        self.downsample = downsample
        self.stride = stride

    def forward(self, x):
        residual = x

        out = self.conv1(x)
        out = self.bn1(out)
        out = self.relu(out)

        out = self.conv2(out)
        out = self.bn2(out)

        if self.downsample is not None:
            residual = self.downsample(x)

        out += residual
        out = self.relu(out)

        return out

class Bottleneck(nn.Module):
    expansion = 4

    def __init__(self, inplanes, planes, stride=1, downsample=None, is_transpose=False):
        super(Bottleneck, self).__init__()
        if is_transpose:
            self.conv2 = nn.ConvTranspose2d(planes, planes, kernel_size=(3,1), stride=stride, padding=(1,0), bias=False)
        else:
            self.conv2 = nn.Conv2d(planes, planes, kernel_size=(3,1), stride=stride, padding=(1,0), bias=False)
        self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False)
        self.bn1 = nn.BatchNorm2d(planes)
        self.bn2 = nn.BatchNorm2d(planes)
        self.conv3 = nn.Conv2d(planes, planes * self.expansion, kernel_size=1, bias=False)
        self.bn3 = nn.BatchNorm2d(planes * self.expansion)
        self.relu = nn.ReLU(inplace=True)
        self.downsample = downsample
        self.stride = stride

    def forward(self, x):
        residual = x

        out = self.conv1(x)
        out = self.bn1(out)
        out = self.relu(out)

        out = self.conv2(out)
        out = self.bn2(out)
        out = self.relu(out)

        out = self.conv3(out)
        out = self.bn3(out)

        if self.downsample is not None:
            residual = self.downsample(x)

        out += residual
        out = self.relu(out)

        return out

class ResCompressionNet(nn.Module):
    def __init__(self, block, layers, r, n_compressed):
        super(ResCompressionNet, self).__init__()
        # r means U_truncated shape[1]
        self.output_dim = (n_compressed,r)
        if r > 512:
            base = 64
        else:
            base = r
        self.inplanes = 2*base
        # self.conv1 = nn.Conv2d(r, 2*r, kernel_size=(7,3), stride=(1,1), padding=(3,1),
        #                        bias=False)
        # self.bn1 = nn.BatchNorm2d(2*r)
        # self.relu = nn.ReLU(inplace=True)
        # self.maxpool = nn.MaxPool2d(kernel_size=3, stride=(1,1), padding=1)
        # self.layer1 = self._make_layer(block, 2*r, layers[0])
        # self.layer2 = self._make_layer(block, 4*r, layers[1], stride=(1,1))

        # self.layer3 = self._make_layer(block, 2*r, layers[2], stride=(1,1))
        # self.layer4 = self._make_layer(block, r, layers[3], stride=(1,1))
        # #TODO: 反卷积？
        self.conv1 = nn.Conv2d(r, 2*base, kernel_size=(7,3), stride=(2,1), padding=(3,1),
                        bias=False)
        self.bn1 = nn.BatchNorm2d(2*base)
        self.relu = nn.ReLU(inplace=True)
        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=(2,1), padding=1)
        self.layer1 = self._make_layer(block, 2*base, layers[0])
        self.layer2 = self._make_layer(block, 4*base, layers[1], stride=(1,1))
        self.layer3 = self._make_transpose_layer(block, 2*base, layers[2], stride=(2,1))
        self.layer4 = self._make_transpose_layer(block, base, layers[3], stride=(2,1))
        #TODO: 反卷积？
        self.avgpool1 = nn.AdaptiveAvgPool2d((n_compressed, 1))
        self.avgpool2 = nn.AdaptiveAvgPool2d((r,1))

        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
            elif isinstance(m, nn.BatchNorm2d):
                nn.init.constant_(m.weight, 1)
                nn.init.constant_(m.bias, 0)

    def _make_transpose_layer(self, block, planes, blocks, stride=1):
        upsample = None
        if stride != 1 or self.inplanes != planes * block.expansion: #or planes!=self.output_dim[1]:
            upsample = nn.Sequential(
                nn.ConvTranspose2d(self.inplanes, planes * block.expansion,
                          kernel_size=1, stride=stride, bias=False),
                nn.BatchNorm2d(planes * block.expansion),
            )

        layers = []
        layers.append(block(self.inplanes, planes, stride, upsample, is_transpose=True))
        self.inplanes = planes * block.expansion
        for i in range(1, blocks):
            layers.append(block(self.inplanes, planes, is_transpose=True))

        return nn.Sequential(*layers)

    def _make_layer(self, block, planes, blocks, stride=1):
        downsample = None
        if stride != 1 or self.inplanes != planes * block.expansion: #or planes!=self.output_dim[1]:
            downsample = nn.Sequential(
                nn.Conv2d(self.inplanes, planes * block.expansion,
                          kernel_size=1, stride=stride, bias=False),
                nn.BatchNorm2d(planes * block.expansion),
            )

        layers = []
        layers.append(block(self.inplanes, planes, stride, downsample))
        self.inplanes = planes * block.expansion
        for i in range(1, blocks):
            layers.append(block(self.inplanes, planes))

        return nn.Sequential(*layers)

    def forward(self, x):
        x = self.conv1(x)
        # print('conv1:',x.size())
        x = self.bn1(x)
        # print('bn1:',x.size())
        x = self.relu(x)
        # print('relu:',x.size())
        x = self.maxpool(x)
        # print('maxpool:',x.size())
        x = self.layer1(x)
        # print('layer1:',x.size())
        x = self.layer2(x)
        # print('layer2:',x.size())
        x = self.layer3(x)
        # print('layer3:',x.size())
        x = self.layer4(x)
        # print('layer4:',x.size())
        x = self.avgpool1(x)
        # print('avgpool1:',x.size())
        x = self.avgpool2(x.view(1,self.output_dim[0],-1,1))
        # print('avgpool2:',x.size())
        x = x.view(self.output_dim[0], self.output_dim[1])
        # print('view::',x.size())

        return x
    @staticmethod
    def add_specific_args(parent_parser: ArgumentParser):
        return parent_parser

def resnet18(r, n_compressed, **kwargs):
    model = ResCompressionNet(BasicBlock, [2,2,2,2], r, n_compressed, **kwargs)
    return model

def resnet34(r, n_compressed, **kwargs):
    model = ResCompressionNet(BasicBlock,[3,4,6,3], r, n_compressed, **kwargs)
    return model

def resnet50(r, n_compressed, **kwargs):
    model = ResCompressionNet(Bottleneck, [3,4,6,3], r, n_compressed, **kwargs)
    return model


import os
import selenium
from selenium import webdriver

with open('sites.txt') as f:
    sites = [line.rstrip() for line in f]

BROWSER = os.getenv("BROWSER", "CHROME")
print("TESTING BROWSER: ", BROWSER)

NUM_RUNS = 10
LOAD_TIMES = []

for site in sites:
    LOAD_TIMES.append({"site": site, "with_extension": [], "default": []})

for _ in range(NUM_RUNS):
    for j, site in enumerate(sites):
        print("site: ", site)

        if BROWSER == "CHROME":
            options = webdriver.ChromeOptions()
            options.add_argument('--no-sandbox')
            options.add_argument('--disable-gpu')
        else:
            options = webdriver.FirefoxOptions()
        options.add_argument('--headless')

        # get load time without extension
        if BROWSER == "CHROME":
            driver = webdriver.Chrome(options=options)
            driver.delete_all_cookies()
        else:
            driver = webdriver.Firefox(options=options)

        try:
            driver.get("https://{}".format(site))
            response_start = driver.execute_script("return window.performance.timing.responseStart")
            dom_complete = driver.execute_script("return window.performance.timing.domComplete")
        except selenium.common.exceptions.WebDriverException as e:
            print(e)

        driver.quit()

        # get load time with extension
        if BROWSER == "CHROME":
            options.add_argument('--load-extension=/usr/src/chrome/')
            driver_extension = webdriver.Chrome(options=options)
            driver_extension.delete_all_cookies()
        else:
            profile = webdriver.FirefoxProfile()
            profile.add_extension(extension='/usr/src/firefox/')
            driver_extension = webdriver.Firefox(options=options, firefox_profile=profile)

        try:
            driver_extension.get("https://{}".format(site))
            response_start_extension = driver_extension.execute_script("return window.performance.timing.responseStart")
            dom_complete_extension = driver_extension.execute_script("return window.performance.timing.domComplete")
        except selenium.common.exceptions.WebDriverException as e:
            print(e)

        driver_extension.quit()

        LOAD_TIMES[j]["with_extension"].append(dom_complete_extension - response_start_extension)
        LOAD_TIMES[j]["default"].append(dom_complete - response_start)

print(LOAD_TIMES)


from xml.etree import ElementTree

from flask import (
    Blueprint, jsonify, render_template, request
)
import requests

from app.response import get_last_day_bucketed_aqi
from app.reading import write_reading

bp = Blueprint('readings', __name__)

@bp.route('/outside', methods=('GET',))
def outside():
    url = 'http://feeds.airnowapi.org/rss/realtime/98.xml'
    print(url)
    response = requests.get(url)
    tree = ElementTree.fromstring(response.content)
    print(tree)
    breakpoint()
    return ''

@bp.route('/data', methods=('GET', 'POST'))
def data():
    if request.method == "GET":
        average_aqi = get_last_day_bucketed_aqi()
        return jsonify(average_aqi)
    else:
        results = request.json
        write_reading(
            device=results['device_id'],
            pmi25=results['pmi2.5'],
            pmi10=results['pmi10']
        )
        return f"wrote {results}", 200

@bp.route('/', methods=('GET',))
def index():
    return render_template('index.html')


import os
import sys  # noqa: F401

import onnx
from onnx import OperatorSetIdProto, TensorProto, helper  # noqa: F401

# Edge that needs to be cut for the split.
# If the edge is feeding into more than one nodes, and not all the nodes belong to the same cut,
# specify those consuming nodes that need to be cut

class CutEdge:
    def __init__(self, edgeId, consumingNodes=None):
        self.edgeId = edgeId
        self.consumingNodes = consumingNodes

def add_expand_type(model, name, type):
    expand_edge = model.graph.value_info.add()
    expand_edge.name = name
    expand_edge.type.CopyFrom(type)

# Add wait/record/send/recv nodes and split the graph into disconnected subgraphs

def split_graph(model, split_edge_groups):
    ms_domain = "com.microsoft"

    new_send_nodes = []
    new_recv_nodes = []

    for cut_index in range(len(split_edge_groups)):
        edgeIds = split_edge_groups[cut_index]  # noqa: N806

        # split the graph based on edgeIds
        upstream_nodes = []
        upstream_nodes_output_index = []
        output_shapes = []
        element_types = []
        for id in edgeIds:
            for node in model.graph.node:
                if len(node.output) >= 1:
                    for i, j in enumerate(node.output):
                        if j == id:
                            upstream_nodes.append(node)
                            upstream_nodes_output_index.append(i)
                            # assuming all tensors are of type float
                            element_types.append(1)
            for info in model.graph.value_info:
                if info.name == id:
                    output_shapes.append(info.type)

        send_input_signal_name = "send_input_signal" + str(cut_index)
        send_signal = model.graph.input.add()
        send_signal.CopyFrom(helper.make_tensor_value_info(send_input_signal_name, onnx.TensorProto.BOOL, None))
        send_signal = helper.make_tensor(send_input_signal_name, TensorProto.BOOL, (), (True,))
        model.graph.initializer.extend([send_signal])

        recv_input_signal_name = "recv_input_signal" + str(cut_index)
        recv_signal = model.graph.input.add()
        recv_signal.CopyFrom(helper.make_tensor_value_info(recv_input_signal_name, onnx.TensorProto.BOOL, None))
        recv_signal = helper.make_tensor(recv_input_signal_name, TensorProto.BOOL, (), (True,))
        model.graph.initializer.extend([recv_signal])

        send_dst_rank_name = "send_dst_rank" + str(cut_index)
        send_dst_rank = model.graph.input.add()
        send_dst_rank.CopyFrom(helper.make_tensor_value_info(send_dst_rank_name, onnx.TensorProto.INT64, None))
        send_dst_rank = helper.make_tensor(send_dst_rank_name, TensorProto.INT64, (), (cut_index + 1,))
        model.graph.initializer.extend([send_dst_rank])

        recv_src_rank_name = "recv_src_rank" + str(cut_index)
        recv_src_rank = model.graph.input.add()
        recv_src_rank.CopyFrom(helper.make_tensor_value_info(recv_src_rank_name, onnx.TensorProto.INT64, None))
        recv_src_rank = helper.make_tensor(recv_src_rank_name, TensorProto.INT64, (), (cut_index,))
        model.graph.initializer.extend([recv_src_rank])

        # output signal from send after cut
        send_output_signal = model.graph.output.add()
        send_output_signal.CopyFrom(
            helper.make_tensor_value_info("send_output_signal" + str(cut_index), onnx.TensorProto.BOOL, None)
        )

        # output signal from receive after cut
        receive_output_signal = model.graph.output.add()
        receive_output_signal.CopyFrom(
            helper.make_tensor_value_info("receive_output_signal" + str(cut_index), onnx.TensorProto.BOOL, None)
        )

        new_send = model.graph.node.add()
        new_send.CopyFrom(
            helper.make_node(
                "Send",
                inputs=[send_input_signal_name, send_dst_rank_name],
                outputs=["send_output_signal" + str(cut_index)],
                tag=0,
                domain=ms_domain,
                element_types=element_types,
                name="send",
            )
        )

        new_receive = model.graph.node.add()
        new_receive.CopyFrom(
            helper.make_node(
                "Recv",
                inputs=[recv_input_signal_name, recv_src_rank_name],
                outputs=["receive_output_signal" + str(cut_index)],
                tag=0,
                domain=ms_domain,
                element_types=element_types,
                name="receive",
            )
        )

        for i in range(len(upstream_nodes)):
            n = upstream_nodes[i]
            idx = upstream_nodes_output_index[i]
            output_type = output_shapes[i]
            output_edge_name = n.output[idx]

            output_nodes = find_all_output_nodes_by_edge(model, output_edge_name)

            # deal with shape inference for newly added edge
            new_send_input_name = output_edge_name + "_send" + str(cut_index)
            add_expand_type(model, new_send_input_name, output_type)

            new_receive_output_name = output_edge_name + "_recv" + str(cut_index)
            add_expand_type(model, new_receive_output_name, output_type)

            # the order of data flow is: node-output -> record -> send -> recv -> wait -> node-input

            new_send.input.extend([output_edge_name])
            new_receive.output.extend([new_receive_output_name])

            for output_node in output_nodes:
                for i in range(len(output_node.input)):  # noqa: PLW2901
                    for edgeId in edgeIds:  # noqa: N806
                        if output_node.input[i] == edgeId:
                            output_node.input[i] = new_receive_output_name

        new_send_nodes.append(new_send)
        new_recv_nodes.append(new_receive)

    model = onnx.shape_inference.infer_shapes(model)

    return new_send_nodes, new_recv_nodes

def find_all_input_nodes(model, node):
    nodes = []
    inputs = []

    if node:
        for inputId in node.input:  # noqa: N806
            nodes.extend([n for n in model.graph.node if inputId in n.output])
            inputs.extend([n for n in model.graph.input if inputId in n.name])
    return nodes, inputs

def find_all_output_nodes(model, node):
    nodes = []
    outputs = []
    if node:
        for outputId in node.output:  # noqa: N806
            nodes.extend([n for n in model.graph.node if outputId in n.input])
            outputs.extend([n for n in model.graph.output if outputId in n.name])
    return nodes, outputs

def find_all_output_nodes_by_edge(model, arg):
    result = [n for n in model.graph.node if arg in n.input]
    return result

# Insert identity nodes to separate same output edge which feeds into different sub-graph.

def add_identity(model, cuttingEdge, newEdgeIdName):
    output_nodes = None
    edgeId = cuttingEdge.edgeId  # noqa: N806
    for node in model.graph.node:
        if len(node.output) >= 1:
            for output in node.output:
                if output == edgeId:
                    output_nodes = find_all_output_nodes_by_edge(model, output)
                    break

    assert output_nodes, "no output node"

    new_identity = model.graph.node.add()
    new_identity.op_type = "Identity"

    new_identity.input.extend([edgeId])
    new_identity.output.extend([newEdgeIdName])

    for i in range(len(output_nodes)):
        for output in output_nodes[i].output:
            if output in cuttingEdge.consumingNodes:
                for j in range(len(output_nodes[i].input)):
                    if output_nodes[i].input[j] == edgeId:
                        output_nodes[i].input[j] = newEdgeIdName

    return new_identity

def insert_identity(model, all_cut_inputs):
    count = 0
    updated_edges = {}
    new_added_identity = []
    split_edge_groups = []
    need_shape_inference = False
    # Sweep the cut edge to see if there are edges feeding into nodes from two sub-graphs. If so,
    # insert identity node after those edges with a new ID to distinguish the rest.
    for cut_input in all_cut_inputs:
        split_edges = []
        for i in cut_input:
            if i.consumingNodes:
                # if this edge has previously been modified, update its edgeId before inserting new identity
                if i.edgeId in updated_edges:
                    i.edgeId = updated_edges[i.edgeId]

                new_edge_name = "identity_output_" + str(count)
                new_added_identity.append(add_identity(model, i, new_edge_name))
                count += 1
                split_edges.append(new_edge_name)
                updated_edges[i.edgeId] = new_edge_name
                need_shape_inference = True
            else:
                split_edges.append(i.edgeId)
        split_edge_groups.append(split_edges)
    return split_edge_groups, new_added_identity, need_shape_inference

# after the graph is split, remove the added identity node because identity op is not registered in gradient builder.

def remove_identity(model, new_added_identity):
    for node in new_added_identity:
        assert node.op_type == "Identity"
        output_nodes = [n for n in model.graph.node if node.output[0] in n.input]
        for output_node in output_nodes:
            for i in range(len(output_node.input)):
                if output_node.input[i] == node.output[0]:
                    output_node.input[i] = node.input[0]

def find_all_connected_nodes(model, node):
    nodes0, inputs = find_all_input_nodes(model, node)
    nodes1, outputs = find_all_output_nodes(model, node)

    connected_nodes = nodes0 + nodes1
    return connected_nodes, inputs, outputs

def get_index(node_list, node):
    found = [i for i, n in enumerate(node_list) if n == node]
    return found[0] if found else None

def get_identity_index_for_deleting(node_list, node):
    for i, n in enumerate(node_list):
        # The node's input name has been changed during send/recv insertion,
        # but it is sufficient to just compare the type and outputs.
        if n.op_type == "Identity" and n.output == node.output:
            return i
    return None

# traverse the graph, group connected nodes and generate subgraph

def generate_subgraph(model, start_nodes, identity_node_list):
    subgraphs = []

    main_graph = onnx.ModelProto()
    main_graph.CopyFrom(model)

    # remove added identity node before copy to subgraph
    identity_node_index = []
    for n in identity_node_list:
        identity_node_index.append(get_identity_index_for_deleting(main_graph.graph.node, n))
    identity_node_index.sort(reverse=True)

    for i in reversed(range(len(main_graph.graph.node))):
        try:
            if i in identity_node_index:
                del main_graph.graph.node[i]
        except Exception:
            print("error deleting identity node", i)

    all_visited_nodes = []
    model_count = len(start_nodes)
    for start in reversed(start_nodes):
        stack0 = [start]

        visited0 = []
        tranversed_node = 0
        inputs0 = []
        outputs0 = []
        while stack0:
            node = stack0.pop()
            if node not in visited0:
                tranversed_node += 1
                visited0.append(node)
                all_visited_nodes.append(node)
                connected_nodes, inputs, outputs = find_all_connected_nodes(main_graph, node)

                stack0 = stack0 + connected_nodes
                inputs0 = inputs0 + inputs
                outputs0 = outputs0 + outputs

        subgraph = onnx.ModelProto()
        subgraph.CopyFrom(main_graph)

        # gather visited nodes
        visited_nodes = []
        for n in visited0:
            visited_nodes.append(get_index(main_graph.graph.node, n))
        visited_nodes.sort(reverse=True)

        # gather visited inputs
        visited_inputs = []
        for n in inputs0:
            visited_inputs.append(get_index(main_graph.graph.input, n))
        visited_inputs.sort(reverse=True)

        # gather visited outputs
        visited_outputs = []
        for n in outputs0:
            visited_outputs.append(get_index(main_graph.graph.output, n))
        visited_outputs.sort(reverse=True)

        for i in reversed(range(len(main_graph.graph.node))):
            try:
                if i not in visited_nodes:
                    del subgraph.graph.node[i]
                else:
                    del main_graph.graph.node[i]
            except Exception:
                print("error deleting node", i)

        for i in reversed(range(len(main_graph.graph.input))):
            try:
                if i not in visited_inputs:
                    del subgraph.graph.input[i]
                else:
                    del main_graph.graph.input[i]
            except Exception:
                print("error deleting inputs", i)

        for i in reversed(range(len(main_graph.graph.output))):
            try:
                if i not in visited_outputs:
                    del subgraph.graph.output[i]
                else:
                    del main_graph.graph.output[i]
            except Exception:
                print("error deleting outputs ", i)

        print("model", str(model_count), " length ", len(subgraph.graph.node))
        subgraphs.append(subgraph)
        model_count -= 1

    print("model", str(model_count), " length ", len(main_graph.graph.node))
    subgraphs.append(main_graph)

    # as the subgraphs were added in reverse order (the last split is added first), reverse the order back before return
    subgraphs.reverse()
    return subgraphs

def main():
    # temporary hard coded the cutting edge structure
    # TODO: move this info to a file (json?) and load the data from there.
    input_model_name = "bert-tiny-uncased_L_3_H_128_A_2_V_30528_S_512_Dp_0.1.onnx"
    stage_count = 3

    cut0_input = {CutEdge("186"), CutEdge("71", {"273", "395"})}
    cut1_input = {CutEdge("308"), CutEdge("71", {"395"})}
    all_cut_inputs = [cut0_input, cut1_input]

    model = onnx.load(input_model_name)
    if len(model.graph.value_info) == 0:
        model = onnx.shape_inference.infer_shapes(model)

    print("original model length ", len(model.graph.node))

    output_model_names = [os.path.splitext(input_model_name)[0] + "_" + str(i) + ".onnx" for i in range(stage_count)]

    split_edge_groups, new_identity, need_shape_inference = insert_identity(model, all_cut_inputs)

    # new edge is being added, need to re-inference shape
    if need_shape_inference:
        model = onnx.shape_inference.infer_shapes(model)

    # after all need-to-be-cut edges identified, split the graph
    new_sends, new_receives = split_graph(model, split_edge_groups)
    remove_identity(model, new_identity)
    sub_graphs = generate_subgraph(model, new_receives, new_identity)

    for i in range(stage_count):
        sub_graphs[i] = onnx.shape_inference.infer_shapes(sub_graphs[i])
        onnx.save(sub_graphs[i], output_model_names[i])
        print("save to file: ", output_model_names[i])

if __name__ == "__main__":
    main()


import urllib.parse
import urllib.request
import json

from flask import Flask, Blueprint, request, flash, redirect, g
from flask_login import current_user, login_user, LoginManager, logout_user, url_for

from .helpers import get_referrer_name, \
    get_sso_center_url, \
    get_sso_center_logout_url, \
    get_sso_center_register_url, \
    get_sso_center_api_ticket_check_url
from .models import UserProfile, AnonymousUser
from .babel import _, lazy_gettext

__all__ = ['make_referrer_query', 'init_sso_login_url', 'init_sso_logout_url', 'init_sso_register_url',
           'sso_login_ticket_check', 'SSOClient'
           ]

def make_referrer_query(referrer=None):
    name = get_referrer_name()
    url = request.referrer if referrer is None or len(referrer.strip()) == 0 else referrer
    dic = {name: url}
    return name, url, dic

def init_sso_login_url(referrer=None):
    """:type query_string: str"""
    sso_center_url = get_sso_center_url()
    dic = make_referrer_query(referrer)[2]
    redirect_url = '{}?{}'.format(sso_center_url, urllib.parse.urlencode(dic))
    return redirect_url

def init_sso_register_url():
    sso_center_url = get_sso_center_register_url()
    dic = make_referrer_query()[2]
    redirect_url = '{}?{}'.format(sso_center_url, urllib.parse.urlencode(dic))
    return redirect_url

def init_sso_logout_url(user_name):
    if user_name is not None:
        logout_url = get_sso_center_logout_url()
        redirect_url = '{}/{}'.format(logout_url, user_name)
        return redirect_url

def sso_login_ticket_check(ticket):
    with urllib.request.urlopen(get_sso_center_api_ticket_check_url()) as f:
        if f.status == 200:
            resp_data = f.read().decode('utf-8')
            resp_data = json.loads(resp_data)
            print('sso_login_ticket_check:', resp_data)
            if resp_data:
                endpoint_url = resp_data['api']
                check_url = urllib.parse.urljoin(get_sso_center_url(), endpoint_url)
                return _sso_login_ticket_check_imp(check_url, ticket)

        return False

def _sso_login_ticket_check_imp(api_url, ticket):
    with urllib.request.urlopen(urllib.parse.urljoin(api_url,ticket)) as f:
        if f.status == 200:
            resp_data = f.read().decode('utf-8')
            resp_data = json.loads(resp_data)
            print('sso-center-ticket-check result:', resp_data)
            return resp_data
        return False

class SSOClient(object):
    def __init__(self, app=None, lm=None):
        self.bp = Blueprint('sso', __name__, url_prefix='/sso')

        if app is not None and lm is not None:
            self.init_app(app, lm)

    def init_app(self, app, lm):
        """
        :type app: Flask
        :type lm: LoginManager
        """
        self.bp.add_url_rule('/register', endpoint='register', view_func=_register, methods=['GET'])
        self.bp.add_url_rule('/login', endpoint='login', view_func=_login, methods=['GET'])
        self.bp.add_url_rule('/logout', endpoint='logout', view_func=_logout, methods=['GET'])

        app.register_blueprint(self.bp)

        # lm.login_view = 'sso.login2'  # 'http://192.168.1.125:9001'
        lm.anonymous_user = AnonymousUser
        lm.login_message = _('Please login to access this page.')

        @lm.unauthorized_handler
        def unauthorized():
            if request.method == 'GET':
                flash(_('Please login to access this page.'))
                # return redirect(login_url('sso.login', request.url, 'url'))
                return redirect(url_for('sso.login', **make_referrer_query(request.url)[2]))
            else:
                return dict(error=True, message=_("Please login for access.")), 403

        @lm.user_loader
        def load_user(user_id):
            user = UserProfile.objects.get(id=user_id)
            return user

        # @lm.request_loader
        # def request_loader(req):
        #     pass
        #
        # @lm.token_loader()
        # def token_loader(req):
        #     pass

        @app.before_request
        def before_request():
            print('current user:', current_user)
            g.user = current_user
            ticket = request.args.get('ticket', None)

            if ticket is not None:
                # print('before_request ticket:', ticket)
                result = sso_login_ticket_check(ticket)
                if result and result['valid']:
                    user = UserProfile.objects(nick_name=result['user']).first()
                    if isinstance(user, UserProfile):
                        login_user(user)
                        print('login user:', user)
                else:
                    logout_user()
                    print('logout user.')
            # else:
            #     print('before_request ticket: None.')

def _login():
    # print('login refer:', request.url)
    if current_user is not None and current_user.is_authenticated:
        return redirect(request.referrer)

    # if redirect from '@lm.unauthorized_handler', request url can get 'url=xxx'
    redirect_url = init_sso_login_url(request.args.get(get_referrer_name()))
    print(redirect_url)
    return redirect(redirect_url)

def _register():
    redirect_url = init_sso_register_url()
    print(redirect_url)
    return redirect(redirect_url)

def _logout():
    redirect_url = init_sso_logout_url(g.user.nick_name)
    logout_user()
    print(redirect_url)
    return redirect(redirect_url)

# def sso_login_required(func):
#     @wraps(func)
#     def decorated_view(*args, **kwargs):
#         ticket = request.args.get('ticket', None)
#         if ticket is not None:
#             print('ticket:', ticket)
#             sso_login_ticket_check()
#         else:
#             print('ticket: None.')
#
#         if current_app.login_manager._login_disabled:
#             return func(*args, **kwargs)
#         elif not current_user.is_authenticated:
#             return current_app.login_manager.unauthorized()
#         return func(*args, **kwargs)
#     return decorated_view


from django.http import HttpResponseRedirect, HttpResponse,\
        HttpResponseNotFound
from django.shortcuts import render_to_response
from django.contrib.auth.decorators import login_required
from django.core.urlresolvers import reverse
from django.template import RequestContext
from django.contrib.contenttypes.models import ContentType

from geoms.forms import GeomForm
from geoms.models import Geom
from olwidget.widgets import MapDisplay, InfoMap

#from vectorformats.Formats import Django, GeoJSON
from django.core import serializers

def list(request, app_label=None, model_name=None, id=None, ):
    ''' List all geoms (ALL, all for model instance, or all for table)

    '''

    # FIXME, I think this could be DRYed out a bit; experimental overloading of the
    # view might not be the coolest thing.
    if not id and not model_name and not app_label:
        geoms = Geom.objects.all()[:15]
        f = []
        for g in geoms:
            f.extend( g.get_info_pairs() )

        map = InfoMap(f,
                    options = {
                        'map_style':{'width':'100%', 'height':'550px',},
                        #'layers': ['google.satellite', 'google.hybrid',  'google.streets' ], # 'google.terrain', ],
                        'default_lat': 44,
                        'default_lon': -72,
                        'default_zoom': 1,
                        'map_div_class':'',
                        'map_div_style': {'width':'100%',},
                    }
        )

        context = { 'map':map,'geoms':geoms, }
        return render_to_response('geoms/all.html', context,\
                context_instance=RequestContext(request))

    elif id and model_name and app_label:
        try:
            ct = ContentType.objects.get(\
                    app_label = app_label,
                    model = model_name)
            obj = ct.get_object_for_this_type( id=id )

        except:
            return HttpResponseRedirect(reverse('geoms_list'))

        geoms = Geom.objects.filter( content_type=ct, object_id=id )
        map = MapDisplay( fields=[p.point for p in geoms],
                map_options = {
                    'map_style':{'width':'100%', 'height':'550px',},
                }
        )

        context = {'geoms':geoms, 'object':obj, 'content_type':ct, 'map':map, }
        return render_to_response('geoms/all.html', context,\
                context_instance=RequestContext(request))

    elif app_label and model_name and not id:
        try:
            ct = ContentType.objects.get(\
                    app_label = app_label,
                    model = model_name)

        except:
            return HttpResponseRedirect(reverse('geoms_list'))

        geoms = Geom.objects.filter(content_type = ct)
        map = MapDisplay( fields=[p.point for p in geoms],
                map_options = {
                    'map_style':{'width':'100%', 'height':'550px',},
                }
        )
        context = {'geoms':geoms, 'content_type':ct, 'map':map,}

        return render_to_response('geoms/all.html', context,\
                context_instance=RequestContext(request))

    else:
        return HttpResponseNotFound()

def detail(request, id):
    ''' Responds with the point and related object information

    '''

    try:
        geom = Geom.objects.get( id=id )

    except Geom.DoesNotExist:
        return HttpResponseRedirect(reverse('geoms_list'))

    map = MapDisplay( fields = geom.geom_fields(),
            options = {
                    'map_div_class':'',
                    'map_div_style': {'width':'100%',},
            }
    )

    return render_to_response('geoms/detail.html', locals(),
                context_instance=RequestContext(request))

@login_required
def delete(request, id):
    ''' can delete a point with a POST from the owner

    '''
    try:
        point = Geom.objects.get(id=id)
    except:
        return HttpResponseNotFound()

    context = {'point':point,}
    if request.user == point.owner:

        if request.method == 'POST':
            point.delete()
            return HttpResponseNotFound()

        else:
            return render_to_response('geoms/confirm_delete.html', context,\
                    context_instance=RequestContext(request))

    else:
        return HttpResponseNotFound()

@login_required
def change(request, id):
    ''' Change the data for a single Geom() obj '''

    try:
        point = Geom.objects.get(id=id)
    except:
        return HttpResponseNotFound()

    if request.method == 'POST' and point.owner == request.user:

        form = GeomForm(request.POST, instance=point)

        if form.is_valid():
            form.save()
            return HttpResponseRedirect(point.get_absolute_url())

    elif point.owner == request.user:
        form = GeomForm( instance=point)
        context = {'point':point, 'form':form, }
        return render_to_response('geoms/change.html', context,\
                context_instance=RequestContext(request) )

    else:
        return HttpResponseNotFound()

@login_required
def add(request, app_label, model_name, id):
    '''add a point to a content_object

    POST the point and the zoom.  The owner is request.user and the related
    obj is received from the url.
    '''
    try:
        ct = ContentType.objects.get(\
                app_label = app_label,
                model = model_name)
        obj = ct.get_object_for_this_type( id=id )

    except:
        return HttpResponseNotFound()

    if request.method == 'POST':
        request.POST.update( {'owner':request.user.id, 'object_id':id,\
                'content_type':ct.id, 'content_obj':obj,} )
        form = GeomForm(request.POST)

        if form.is_valid():
            form.save()

            #try:
            #    return HttpResponseRedirect(request.META['HTTP_REFERER'])

            #except:

            try:
                return HttpResponseRedirect(obj.get_absolute_url())

            except:
                return HttpResponseRedirect(reverse('geoms_list'))

    else:
        form = GeomForm()

    context = {'form':form, 'object':obj, 'content_type':ct, }
    context.update(locals())

    return render_to_response('geoms/add.html', context,\
            context_instance = RequestContext(request))


#########################################################################
## Project: Block and date extraction                                  ##
## Date: January 2022                                                  ##
## Author: Tono / Sung Wook Chung (Transcranial Solutions)             ##
## transcranial.solutions@gmail.com                                    ##
##                                                                     ##
## This programme is distributed in the hope that it will be useful,   ##
## but WITHOUT ANY WARRANTY, without even the implied warranty of      ##
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the       ##
## GNU General Public License for more details.                        ##
#########################################################################

# Extract

# import json library
# import urllib
# from urllib.request import Request, urlopen
# import json
import pandas as pd
import numpy as np
# import matplotlib.pyplot as plt
# from pandas.core.groupby.generic import DataFrameGroupBy
# import pylab as pl
import os
from functools import reduce
from concurrent.futures import ThreadPoolExecutor, as_completed
import concurrent.futures
from iconsdk.icon_service import IconService
from iconsdk.providers.http_provider import HTTPProvider
from iconsdk.wallet.wallet import KeyWallet
# from iconsdk.builder.call_builder import CallBuilder
# from typing import Union
from time import time
from datetime import date, datetime, timedelta
from tqdm import tqdm
from functools import reduce
import re
import itertools
import random
import codecs

desired_width=320
pd.set_option('display.width', desired_width)
pd.set_option('display.max_columns',10)

workers = 8

start_block = 1
end_block = 100

currPath = os.getcwd()
if not "06_wallet_ranking" in currPath:
    projectPath = os.path.join(currPath, "06_wallet_ranking")
    if not os.path.exists(projectPath):
        os.mkdir(projectPath)
else:
    projectPath = currPath

dataPath = os.path.join(projectPath, "data")
if not os.path.exists(dataPath):
    os.mkdir(dataPath)

resultPath = os.path.join(projectPath, "results")
if not os.path.exists(resultPath):
    os.mkdir(resultPath)

walletPath = os.path.join(currPath, "wallet")
if not os.path.exists(walletPath):
    os.mkdir(walletPath)

def insert_str(string="YYYY-WW", index=4, timeinterval=' week') -> str:
    """
    Inserting strings between strings
    """
    return string[:index] + timeinterval + string[index:]

## value extraction function
def extract_values(obj, key):
    # Pull all values of specified key from nested JSON
    arr = []
    def extract(obj, arr, key):
        # Recursively search for values of key in JSON tree
        if isinstance(obj, dict):
            for k, v in obj.items():
                if isinstance(v, (dict, list)):
                    extract(v, arr, key)
                elif k == key:
                    arr.append(v)
        elif isinstance(obj, list):
            for item in obj:
                extract(item, arr, key)
        return arr

    results = extract(obj, arr, key)
    if not results:
        # results=''
        results=np.nan
    return results

## value extraction function
def extract_values_no_params(obj, key):
    # Pull all values of specified key from nested JSON
    arr = []
    def extract(obj, arr, key):
        # Recursively search for values of key in JSON tree
        if isinstance(obj, dict):
            for k, v in obj.items():
                if k != "params" and v != 'txHash':
                    if isinstance(v, (dict, list)):
                        extract(v, arr, key)
                    elif k == key:
                        arr.append(v)
        elif isinstance(obj, list):
            for item in obj:
                extract(item, arr, key)
        return arr

    results = extract(obj, arr, key)
    if not results:
        # results=''
        results=np.nan
    return results

def deep_get_imps(data, key: str):
    split_keys = re.split("[\\[\\]]", key)
    out_data = data
    for split_key in split_keys:
        if split_key == "":
            return out_data
        elif isinstance(out_data, dict):
            out_data = out_data.get(split_key)
        elif isinstance(out_data, list):
            try:
                sub = int(split_key)
            except ValueError:
                return None
            else:
                length = len(out_data)
                out_data = out_data[sub] if -length <= sub < length else None
        else:
            return None
    if not out_data:
        # out_data=''
        out_data=np.nan
    return out_data

def deep_get(dictionary, keys):
    return reduce(deep_get_imps, keys.split("."), dictionary)

#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Local network ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
 # localhost
icon_service = IconService(HTTPProvider("http://127.0.0.1:9000/api/v3", request_kwargs={"timeout": 60}))
# block = icon_service.get_block(38000000)

## Creating Wallet if does not exist (only done for the first time)
tester_wallet = os.path.join(walletPath, "test_keystore_1")

if os.path.exists(tester_wallet):
    wallet = KeyWallet.load(tester_wallet, "abcd1234*")
else:
    wallet = KeyWallet.create()
    wallet.get_address()
    wallet.get_private_key()
    wallet.store(tester_wallet, "abcd1234*")

tester_address = wallet.get_address()

#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ block Info ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#

# blockInfo=24895857
def get_block_df(blockInfo):
    blockInfo = icon_service.get_block(blockInfo)

    address = extract_values_no_params(blockInfo, "from")
    to_address = extract_values_no_params(blockInfo, "to")

    try:
        address[0:0] = to_address
        unique_address = list(set(address))
        block_df = pd.DataFrame(data=unique_address, columns=['address'])
        return block_df
    except:
        pass

#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
# collecting contact info using multithreading

block_of_interest = range(start_block, end_block)

def run_block():
    start = time()
    with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor:
        myblock = list(tqdm(executor.map(get_block_df, block_of_interest), total=len(block_of_interest)))

    block_df = pd.concat(myblock).reset_index(drop=True).drop_duplicates()
    print(f'Time taken: {time() - start}')

    return block_df

block_df = run_block()

block_df.to_csv(os.path.join(dataPath, 'address_' + str(start_block) + '_' + str(end_block) + '.csv'), index=False)

def run_all():
    if __name__ == "__main__":
        run_all()


import sympy as sp
from sympy.abc import a
import autograd.numpy as np
from pint import UnitRegistry
ureg = UnitRegistry()

class Var(sp.core.Symbol):
    def __new__(cls, name, value=None, unit=None):
        out = super().__new__(cls, name)
        out.varval = value
        out.varunit = unit
        return out

def get_unit_multi(unit):
    return unit.to_base_units().magnitude

def var_generator(m):
    def wrapped(name, value=None, unit=None):
        unit = ureg(unit)
        if isinstance(value, sp.Basic):
            free_symbols = list(value.free_symbols)
            rhs = value
            rhsfx = sp.lambdify(free_symbols, rhs, modules=np)
            rhs_unit = rhsfx(*(free_symbol.varunit
                for free_symbol in free_symbols))
            convert = np.array([get_unit_multi(free_symbol.varunit) for
                    free_symbol in free_symbols])
            if unit:
                assert(unit.dimensionality == rhs_unit.dimensionality)
                conversion_unit = unit
            else:
                conversion_unit = ureg.Quantity(1, rhs_unit.units)

            factor = get_unit_multi(conversion_unit)

            rhsfx_corrected = lambda *args: rhsfx(*(
                convert*np.array(args).flatten()))/factor
            m.add_eq(name, rhsfx_corrected, tuple([free_symbol.name
                for free_symbol in free_symbols]))
            return Var(name, unit=unit)
        else:
            if value:
                if unit == ureg('deg'):
                    value = (value*unit).to(ureg('rad')).magnitude
                    unit = ureg('rad')
                m.set_var(name, value)
            return Var(name, value, unit)
    return wrapped