import os
import shutil
import subprocess
import tempfile
from typing import List

from phrasetree.tree import Tree

from elit.metrics.f1 import F1
from elit.metrics.metric import Metric
from elit.utils.io_util import get_resource, run_cmd
from elit.utils.log_util import cprint

class EvalbBracketingScorer(Metric):
    """
    This class uses the external EVALB software for computing a broad range of metrics
    on parse trees. Here, we use it to compute the Precision, Recall and F1 metrics.
    You can download the source for EVALB from here: <https://nlp.cs.nyu.edu/evalb/>.

    Note that this software is 20 years old. In order to compile it on modern hardware,
    you may need to remove an `include <malloc.h>` statement in `evalb.c` before it
    will compile.

    AllenNLP contains the EVALB software, but you will need to compile it yourself
    before using it because the binary it generates is system dependent. To build it,
    run `make` inside the `allennlp/tools/EVALB` directory.

    Note that this metric reads and writes from disk quite a bit. You probably don't
    want to include it in your training loop; instead, you should calculate this on
    a validation set only.

    # Parameters

    evalb_directory_path : `str`, required.
        The directory containing the EVALB executable.
    evalb_param_filename : `str`, optional (default = `"COLLINS.prm"`)
        The relative name of the EVALB configuration file used when scoring the trees.
        By default, this uses the nk.prm configuration file which comes with LAL-Parser.
        This configuration ignores POS tags, S1 labels and some punctuation labels.
    evalb_num_errors_to_kill : `int`, optional (default = `"10"`)
        The number of errors to tolerate from EVALB before terminating evaluation.
    """

    def __init__(
            self,
            evalb_directory_path: str = None,
            evalb_param_filename: str = "nk.prm",
            evalb_num_errors_to_kill: int = 10,
    ) -> None:
        if not evalb_directory_path:
            evalb_directory_path = get_resource('https://github.com/KhalilMrini/LAL-Parser/archive/master.zip#EVALB/')
        self._evalb_directory_path = evalb_directory_path
        self._evalb_program_path = os.path.join(evalb_directory_path, "evalb")
        self._evalb_param_path = os.path.join(evalb_directory_path, evalb_param_filename)
        self._evalb_num_errors_to_kill = evalb_num_errors_to_kill

        self._header_line = [
            "ID",
            "Len.",
            "Stat.",
            "Recal",
            "Prec.",
            "Bracket",
            "gold",
            "test",
            "Bracket",
            "Words",
            "Tags",
            "Accracy",
        ]

        self._correct_predicted_brackets = 0.0
        self._gold_brackets = 0.0
        self._predicted_brackets = 0.0

    def __call__(self, predicted_trees: List[Tree], gold_trees: List[Tree]) -> None:  # type: ignore
        """
        # Parameters

        predicted_trees : `List[Tree]`
            A list of predicted NLTK Trees to compute score for.
        gold_trees : `List[Tree]`
            A list of gold NLTK Trees to use as a reference.
        """
        if not os.path.exists(self._evalb_program_path):
            cprint(f"EVALB not found at {self._evalb_program_path}.  Attempting to compile it.")
            EvalbBracketingScorer.compile_evalb(self._evalb_directory_path)

            # If EVALB executable still doesn't exist, raise an error.
            if not os.path.exists(self._evalb_program_path):
                compile_command = (
                    f"python -c 'from allennlp.training.metrics import EvalbBracketingScorer; "
                    f'EvalbBracketingScorer.compile_evalb("{self._evalb_directory_path}")\''
                )
                raise RuntimeError(
                    f"EVALB still not found at {self._evalb_program_path}. "
                    "You must compile the EVALB scorer before using it."
                    " Run 'make' in the '{}' directory or run: {}".format(
                        self._evalb_program_path, compile_command
                    )
                )
        tempdir = tempfile.mkdtemp()
        gold_path = os.path.join(tempdir, "gold.txt")
        predicted_path = os.path.join(tempdir, "predicted.txt")
        with open(gold_path, "w") as gold_file:
            for tree in gold_trees:
                gold_file.write(f"{tree.pformat(margin=1000000)}\n")

        with open(predicted_path, "w") as predicted_file:
            for tree in predicted_trees:
                predicted_file.write(f"{tree.pformat(margin=1000000)}\n")

        command = [
            self._evalb_program_path,
            "-p",
            self._evalb_param_path,
            "-e",
            str(self._evalb_num_errors_to_kill),
            gold_path,
            predicted_path,
        ]
        completed_process = run_cmd(' '.join(command))

        _correct_predicted_brackets = 0.0
        _gold_brackets = 0.0
        _predicted_brackets = 0.0

        for line in completed_process.split("\n"):
            stripped = line.strip().split()
            if len(stripped) == 12 and stripped != self._header_line:
                # This line contains results for a single tree.
                numeric_line = [float(x) for x in stripped]
                _correct_predicted_brackets += numeric_line[5]
                _gold_brackets += numeric_line[6]
                _predicted_brackets += numeric_line[7]

        shutil.rmtree(tempdir)

        self._correct_predicted_brackets += _correct_predicted_brackets
        self._gold_brackets += _gold_brackets
        self._predicted_brackets += _predicted_brackets

    def get_metric(self):
        """
        # Returns

        The average precision, recall and f1.
        """

        return F1(self._predicted_brackets, self._gold_brackets, self._correct_predicted_brackets)

    def reset(self):
        self._correct_predicted_brackets = 0.0
        self._gold_brackets = 0.0
        self._predicted_brackets = 0.0

    @staticmethod
    def compile_evalb(evalb_directory_path: str = None):
        os.system("cd {} && make && cd ../../../".format(evalb_directory_path))
        run_cmd('chmod +x ' + os.path.join(evalb_directory_path, "evalb"))

    @staticmethod
    def clean_evalb(evalb_directory_path: str = None):
        return run_cmd("rm {}".format(os.path.join(evalb_directory_path, "evalb")))

    @property
    def score(self):
        return self.get_metric().prf[-1]

    def __repr__(self) -> str:
        return str(self.get_metric())

def main():
    tree1 = Tree.fromstring("(S (NP (D the) (N dog)) (VP (V chased) (NP (D the) (N cat))))")
    tree2 = Tree.fromstring("(S (NP (D the) (N dog)) (VP (V chased) (NP (D the) (N cat))))")
    evalb_scorer = EvalbBracketingScorer()
    evalb_scorer([tree1], [tree2])
    metrics = evalb_scorer.get_metric()
    assert metrics.prf == (1.0, 1.0, 1.0)

    tree1 = Tree.fromstring("(S (VP (D the) (NP dog)) (VP (V chased) (NP (D the) (N cat))))")
    tree2 = Tree.fromstring("(S (NP (D the) (N dog)) (VP (V chased) (NP (D the) (N cat))))")
    evalb_scorer = EvalbBracketingScorer()
    evalb_scorer([tree1], [tree2])
    metrics = evalb_scorer.get_metric()
    assert metrics.prf == (0.75, 0.75, 0.75)

    tree1 = Tree.fromstring("(S (VP (D the) (NP dog)) (VP (V chased) (NP (D the) (N cat))))")
    tree2 = Tree.fromstring("(S (NP (D the) (N dog)) (VP (V chased) (NP (D the) (N cat))))")
    evalb_scorer = EvalbBracketingScorer()
    evalb_scorer([tree1, tree2], [tree2, tree2])
    metrics = evalb_scorer.get_metric()
    assert metrics.prf == (0.875, 0.875, 0.875)

if __name__ == '__main__':
    main()


"""A modified BFS algorithm for the Android Bubble Sort Puzzle Solver."""

from collections import deque

from solver.trie import Trie
from solver.state import is_solved_state
from solver.state import count_finished_tubes
from solver.state import get_next_states

class QuantizedDoubleEndedPriorityQueue:
    """A Quantized version of the deque data structure."""

    def __init__(self):
        """Create a QuantizedDoubleEndedPriorityQueue."""
        self.d = {}
        self.count = 0

    def add(self, rank, state, moves):
        """Add a state to QuantizedDoubleEndedPriorityQueue."""
        if rank not in self.d:
            self.d[rank] = deque()
        self.d[rank].append((state, moves))
        self.count += 1

    def get(self):
        """Return the next state to be checked."""
        rank = max(self.d.keys())

        ret = self.d[rank].popleft()

        if len(self.d[rank]) == 0:
            del self.d[rank]

        self.count -= 1
        return ret

def bfs(initial_state):
    """Run the modified BFS algorithm."""
    states = QuantizedDoubleEndedPriorityQueue()

    rank = count_finished_tubes(initial_state)
    states.add(rank, initial_state, [])

    visited_states = Trie()

    while states.count > 0:
        # print("#" * (len(states) // 100))
        # print("#" * (len(visited_state_strings) // 10))

        state, moves = states.get()

        # print("")
        # print("")
        # print(f"BFS state is: {state}")

        if visited_states.contains(state):
            continue

        if is_solved_state(state):
            return moves

        visited_states.add(state)

        for next_state, next_moves in get_next_states(state, moves):
            if visited_states.contains(next_state):
                continue

            rank = count_finished_tubes(next_state)
            states.add(rank, next_state, next_moves)


from model import db_connection
from helper import id_generator,paginated_results
from flask import Flask, request

class Contacts:
    def __init__(self):
        self.db=db_connection.Database()

    def record_exists(self,field,data):
        count = self.db.get_doc_count("contacts",field,data)
        if count:
            return True
        return False

    def generate_id(self):
        id = id_generator.ran_gen()
        while True:
            if self.record_exists("contact_id",id):
                id = id_generator.ran_gen()
            else:
                return id

    def create_contact(self,input):
        output ={"status": "" , "message" : ""}
        try:
            name=input['name']
            email=input['email']
            user=input['user']

            if self.record_exists("email",email):
                output["status"] = "error"
                output["message"] = "Email Exists"

            else:
                data_to_insert = {}
                data_to_insert['contact_id'] = self.generate_id()
                data_to_insert['name'] = name
                data_to_insert['email'] = email

                self.db.insert_one_data("contacts",data_to_insert)
                self.db.insert_one_to_array("users","user",user,"contacts",data_to_insert['contact_id'])
                output["status"] = "success"

        except:
            output["status"] = "success"

        return output

    def remove_contact(self,input):
        output ={"status": "" , "message" : ""}
        try:
            user=input['user']
            contact_id=input['contact_id']
            self.db.delete_one_from_array("users","user",user,"contacts",contact_id)
            output["status"] = "success"
        except :
            output["status"] = "error"

        return output

    def list_contact(self,input):
        output ={"status": "" , "message" : ""}

        try:
            try:
                page = int(request.args.get("page"))
            except:
                page = 1

            try:
                limit = int(request.args.get("limit"))
            except:
                limit = 10

            user=input['user']
            query={"user":user}
            contact_ids = self.db.get_values("users",query,["contacts"],["_id"])[0]['contacts']

            query={"contact_id":{"$in":contact_ids}}
            contacts = self.db.get_values("contacts",query,["contact_id","name","email"],["_id"])

            output["results"]= paginated_results.paginated_results(contacts,page,limit)
            output["status"] = "success"
        except:
            output["status"] = "error"

        return output

    def search_contact(self,input):
        output ={"status": "" , "message" : ""}

        try:
            search_string = input['search_string']
            output["results"] = self.db.search("contacts",search_string)
            output["status"] = "success"
        except:
            output["status"] = "error"

        return output

    def update_contact(self,input):
        output ={"status": "" , "message" : ""}

        try:
            fields_to_update = input['fields_to_update']
            contact_id = input['contact_id']

            new_fields = {}
            for field in fields_to_update:
                new_fields[field] = input[field]
            self.db.update_one("contacts","contact_id",contact_id,new_fields)
            output["status"] = "success"
        except:
            output["status"] = "error"

        return  output

    def add_contacts(self,input):
        output ={"status": "" , "message" : ""}

        try:
            contact_ids = input['contact_ids']
            user=input['user']
            self.db.insert_many_to_array("users","user",user,"contacts",contact_ids)
            output["status"] = "success"
        except :
            output["status"] = "error"

        return output


# --------------------------------
# Name: plot_sbcape_loop.py
# Author: Robert M. Frost
# NOAA Global Systems Laboratory
# Created: 26 June 2023
# Purpose: Loop to plot 2m dew point
# and wind barb comparisons at
# different times during forecast runs
# --------------------------------
from UFSutils import read_grib
import matplotlib.pyplot as plt
from matplotlib import rc
import cartopy.crs as ccrs
import cartopy.feature as cpf
import seaborn
import numpy as np
import geopandas as gpd
# --------------------------------
# settings

# date being plot
date = "2023041912" #YYYMMDDHH
# hour forecast was initialized (UTC)
init = 12
# directory where hrrr grib data are located
dgrib_h = f"/scratch2/BMC/fv3lam/Robby.Frost/expt_dirs/{date}_3km_hrrrphys/{date}/postprd/"
# directory where rap grib data are located
dgrib_r = f"/scratch2/BMC/fv3lam/Robby.Frost/expt_dirs/{date}_3km_rapphys/{date}/postprd/"
# natlev or prslev
nat_prs = "natlev"
# message number for dew point
mn_td2m = 1358
# message number for u at 10m
mn_u10 = 1364
# message number for v at 10m
mn_v10 = 1365
# directory for figure to be output
figdir = f"/scratch2/BMC/fv3lam/Robby.Frost/figures/{date}/td2m/"
# --------------------------------
# plotting setup
rc('font',weight='normal',size=12.5)
# rc('text',usetex='True')
rc('figure',facecolor='white')
# --------------------------------
# NWS dew point colorbar

import matplotlib.colors as colors
a = np.array([0,10,20,30,40,45,50,55,60,65,70,75,80])
# Normalize the bin between 0 and 1 (uneven bins are important here)
norm = [(float(i)-min(a))/(max(a)-min(a)) for i in a]
# Color tuple for every bin
C = np.array([[59,34,4],
            [84,48,5],
            [140,82,10],
            [191,129,45],
            [204,168,84],
            [223,194,125],
            [230,217,181],
            [211,235,231],
            [169,219,211],
            [114,184,173],
            [49,140,133],
            [1,102,95],
            [0,60,48],
            [0,41,33]])
# Create a tuple for every color indicating the normalized position on the colormap and the assigned color.
COLORS = []
for i, n in enumerate(norm):
    COLORS.append((n, np.array(C[i])/255.))
# Create the colormap
cmap = colors.LinearSegmentedColormap.from_list("dewpoint", COLORS)
# --------------------------------
# loop over time

for hr in range(0,37):
    print(f"Hour {hr}")

    # read in dew point
    hrrr, td2m_h, lat, lon, valid_date = read_grib(init, hr, dgrib_h, nat_prs, mn_td2m, ret_type=0)
    rap, td2m_r, lat, lon, valid_date = read_grib(init, hr, dgrib_r, nat_prs, mn_td2m, ret_type=0)
    # convert to fahrenheit (superior unit of temperature)
    td2m_h = (td2m_h.values - 273.15) * (9/5) + 32
    td2m_r = (td2m_r.values - 273.15) * (9/5) + 32

    # read in 10m wind
    u10_h = hrrr[mn_u10].values
    v10_h = hrrr[mn_v10].values
    u10_r = rap[mn_u10].values
    v10_r = rap[mn_v10].values

    # convert 10m wind to knots
    u10_h = u10_h * 1.944
    v10_h = v10_h * 1.944
    u10_r = u10_r * 1.944
    v10_r = v10_r * 1.944
    # --------------------------------
    # Plot dew point comparison
    print(f"Creating 1 x 2 Td2m Plot")

    # Define your custom colorbar bounds
    cbar_min = 0
    cbar_max = 80.1
    # levels for sbcape to be plot
    clevs = np.arange(cbar_min, cbar_max, 2)

    # create plot
    fig, ax = plt.subplots(ncols=2, subplot_kw={'projection': ccrs.PlateCarree()},
                        figsize=(16,10), constrained_layout=True)

    # plot HRRR
    c0 = ax[0].contourf(lon, lat, td2m_h, clevs,
                        transform=ccrs.PlateCarree(),
                        cmap=cmap, extend="both")
    # plot RAP
    c1 = ax[1].contourf(lon, lat, td2m_r, clevs,
                        transform=ccrs.PlateCarree(),
                        cmap=cmap, extend="both")

    # mapping
    plt_area = [-101, -94, 30, 37.5] # W, E, S, N
    for i, iax in enumerate(ax):
        iax.coastlines()
        iax.add_feature(cpf.BORDERS)
        iax.add_feature(cpf.STATES)
        iax.set_extent(plt_area)
        # Load the json file with county coordinates
        geoData = gpd.read_file('https://raw.githubusercontent.com/holtzy/The-Python-Graph-Gallery/master/static/data/US-counties.geojson')
        geoData.plot(ax=iax, color="none", lw=0.3, aspect=1)

    # set title
    ax[0].set_title(f"No-GF F0{hr},  Valid {valid_date} UTC")
    ax[1].set_title(f"GF F0{hr},  Valid {valid_date} UTC")

    # Add colorbar
    cbar = fig.colorbar(c1, ax=ax, orientation='horizontal', extend=True, pad=0.03, aspect=50)
    cbar.set_label('2m Dew Point Temperature [$^{\circ}$F]')
    cbar.set_ticks(np.arange(cbar_min, cbar_max, 10))

    # Wind barbs
    spacing=25 #barbspacing (smaller if zoomed in)
    ax[0].barbs(lon[::spacing,::spacing], lat[::spacing,::spacing],
                u10_h[::spacing,::spacing], v10_h[::spacing,::spacing],
                length=6)
    ax[1].barbs(lon[::spacing,::spacing], lat[::spacing,::spacing],
                u10_r[::spacing,::spacing], v10_r[::spacing,::spacing],
                length=6)

    # save and close figure
    figdir_full = f"{figdir}td2m_sidebyside_f{hr}.png"
    print(f"Saving figure to {figdir_full}")
    plt.savefig(figdir_full)
    plt.close()
    print("Finished plotting 1 x 2 Td2m!")
    # --------------------------------
    # Plot dew point comparison
    print("Creating Td2m Difference Plot!")

    # Define your custom colorbar bounds
    cbar_min = -30
    cbar_max = 30.1
    # contour levels
    clevs = np.linspace(cbar_min, cbar_max, 50)
    # color palette
    colors = seaborn.color_palette("seismic", as_cmap=True)

    # create plot
    fig, ax = plt.subplots(subplot_kw={'projection': ccrs.PlateCarree()},
                        figsize=(10,6.5), constrained_layout=True)

    # plot HRRR - RAP
    c0 = ax.contourf(lon, lat, td2m_h - td2m_r,
                    clevs, transform=ccrs.PlateCarree(),
                    cmap=colors, extend="both")

    # mapping
    plt_area = [-101, -94, 33.5, 37.5] # W, E, S, N
    ax.coastlines()
    ax.add_feature(cpf.BORDERS)
    ax.add_feature(cpf.STATES)
    ax.set_extent(plt_area)
    # Load the json file with county coordinates
    geoData = gpd.read_file('https://raw.githubusercontent.com/holtzy/The-Python-Graph-Gallery/master/static/data/US-counties.geojson')
    geoData.plot(ax=ax, color="none", lw=0.3, aspect=1)

    # set title
    ax.set_title(f"HRRR - RAP F0{hr},  Valid {valid_date} UTC")

    # Add colorbar
    cbar = fig.colorbar(c0, ax=ax, orientation='horizontal', extend=True, pad=0.03, aspect=50)
    cbar.set_label('HRRR - RAP 2m Dew Point Temperature [$^{\circ}$F]')
    cbar.set_ticks(np.arange(cbar_min, cbar_max, 5))

    # save and close figure
    figdir_full = f"{figdir}td2m_diff_f{hr}.png"
    print(f"Saving figure to {figdir_full}")
    plt.savefig(figdir_full)
    plt.close()
    print(f"Finished with hour {hr}! \n")


""" Livestock Animal Pens """

#Django
from django.db import models

ORIENTATION_CHOICES = (
    ('Norte','Norte'),
    ('Sur','Sur'),
    ('Este','Este'),
    ('Oeste','Oeste')
)

class LivestockAnimalPens(models.Model):
    """ Modelo de Corrales de animales
    de la produccion ganadera """

    production_livestock = models.ForeignKey(
        "producer.ProductionLivestock",
        related_name="livestock_animal_pens",
        on_delete=models.CASCADE
        )
    orientation = models.CharField(max_length=20, blank=True, null=True)
    building_material = models.CharField(max_length=50, blank=True, null=True)
    roof_material = models.CharField(max_length=30, blank=True, null=True)
    foor_material = models.CharField(max_length=30, blank=True, null=True)
    surface = models.FloatField(default=0)
    num_animals = models.PositiveIntegerField(default=0)
    lat = models.FloatField(default=0)
    lng = models.FloatField(default=0)


from mpl_toolkits.mplot3d import Axes3D
import matplotlib.pyplot as plt
import numpy as np
import os
import time

fig = plt.figure()
plt.ion()
ax = fig.add_subplot(111, projection='3d')
ax.grid(False)

for root, folder, files in os.walk("./results/features"):
    for file in sorted(files):
        if ".npy" not in file:
            continue

        p = np.load(os.path.join("./results/features",file))
        p = np.reshape(p, (13,3))
        ax.clear()
        ax.grid(False)
        ax.set_xlim([-1, 1])
        ax.set_ylim([-1, 1])
        ax.set_zlim([-1, 1])

        ax.scatter(p[:,0],p[:,1],p[:,2])
        plt.draw()
        plt.pause(1)
        plt.show()


#!/usr/bin/env python
from __future__ import absolute_import
from __future__ import print_function
import numpy as np
import pandas as pd
import re
import util
import os
import entrez as ez
from xgmml import *
import parallel
import db
from six.moves import range
import setting

class MCODECluster(Network):

    def __init__(self, network, seedNode=None, score=0.0):
        self.seedNode=seedNode
        self.score=score or 0.0
        super(MCODECluster, self).__init__(network)

    def __str__(self):
        s="SeedNode: "+self.seedNode+"\n"
        s+="Score: "+str(self.score)+"\n"
        s+=super(MCODECluster, self).__str__()
        return s

class MCODE(Network):

    #the parameters used for this instance of the algorithm
    params = {
        'includeLoops': False,
        'degreeCutoff': 2,
        'kCore': 2, # kCore must be greater than 1
        'maxDepthFromStart': 100,
        'nodeScoreCutoff': 0.2,
        'fluff': False,
        'haircut': True,
        'fluffNodeDensityCutoff': 0.1
    }

    def get_max_score(self):
        return self.C_nodeByScore[0][0]

    @staticmethod
    def calc_density(gpInputGraph, includeLoops=False):
        if (gpInputGraph.is_empty()): return -1.0
        loopCount=0
        if includeLoops:
            S_node=gpInputGraph.nodes()
            for node in S_node:
                if (gpInputGraph.are_neighbors(node, node)): loopCount+=1
        n=gpInputGraph.nof_nodes()
        possibleEdgeNum = n**2
        actualEdgeNum = gpInputGraph.nof_edges() - loopCount
        return actualEdgeNum*1.0/possibleEdgeNum

    def score_network(self, network):
        numNodes = network.nof_nodes()
        density = MCODE.calc_density(network, self.params['includeLoops'])
        score = density * numNodes
        return score

    @staticmethod
    def get_KCore(gpInputGraph, k):
        if (gpInputGraph is None or gpInputGraph.is_empty()):
            util.error_msg("GetKCore(): no input network!")
        #filter all nodes with degree less than k until convergence
        firstLoop = True
        gpOutputGraph = None
        while True:
            numDeleted = 0
            S_node = gpInputGraph.nodes()
            alCoreNodes=[node for node in S_node if len(gpInputGraph.data[node]) >= k]
            # ZHOU 3/16/2018
            if len(alCoreNodes)<k: return None
            #
            if (len(S_node)>len(alCoreNodes) or firstLoop):
                gpOutputGraph = gpInputGraph.subnetwork(alCoreNodes)
                if (gpOutputGraph.is_empty()): return None
                #iterate again, but with a new k-core input graph
                gpInputGraph = gpOutputGraph
                firstLoop = False
            else:
                #stop the loop
                break
        return gpOutputGraph

    def get_highest_KCore(self, gpInputGraph):
        S_md5=[]
        if self.l_cache:
            s_md5=gpInputGraph.node_MD5()
            S_md5.append(s_md5)
            if s_md5 in self.cache_kcore:
                k=self.cache_kcore[s_md5]
                gpPrevCore=MCODE.get_KCore(gpInputGraph, k)
                return {'k':self.cache_kcore[s_md5], 'network':gpPrevCore}

        gpCurCore=gpPrevCore=None
        ### ZHOU 3/16/2018 tries to speed things up, find the max possible k
        R_degree=np.array([gpInputGraph.degree(x) for x in gpInputGraph.nodes()])
        if len(R_degree)==0: return {'k':0, 'network':gpPrevCore}
        # https://stackoverflow.com/questions/26984414/efficiently-sorting-a-numpy-array-in-descending-order/26984520
        R_degree[::-1].sort()
        # if k is a max, then there must be at least k nodes with degrees >= k
        lb=ub=R_degree[-1]
        if lb==0: exit()
        # all degrees in tmp are possible k, so we use max(tmp)
        tmp=R_degree[R_degree<=np.arange(1,len(R_degree)+1)]
        if len(tmp)>0:
            ub=max(tmp)
            # Be aware if candidates [5 5 5 2 2 2], max k can be 3, not in the candidate list
            # the above tmp will give ub=2, so we need to find the closest failure
            tmp2=R_degree[R_degree>ub]
            if len(tmp2): ub=min(tmp2)-1
        gpPrevCore=MCODE.get_KCore(gpInputGraph, lb)
        while (lb<ub): # lb is already a solution, ub has not been explored yet
            k=max(lb+1, int(lb*0.3+ub*0.7)) # empirically seems a bit better to bias towards ub
            #print("Try: ", k, "[", lb, ub, "]")
            gpCurCore = MCODE.get_KCore(gpInputGraph, k)
            if gpCurCore is None or gpCurCore.is_empty():
                #print("fail")
                ub=k-1
            else:
                gpPrevCore = gpCurCore
                gpInputGraph=gpCurCore # let's shrink the search space
                # use cache to avoid recomputing.
                if self.l_cache:
                    s_md5=gpInputGraph.node_MD5()
                    S_md5.append(s_md5)
                    if s_md5 in self.cache_kcore:
                        k=self.cache_kcore[s_md5]
                        gpPrevCore=MCODE.get_KCore(gpInputGraph, k)
                        lb=ub=k
                lb=k
        #while True:
        #    gpCurCore = MCODE.get_KCore(gpInputGraph, i)
        #    if gpCurCore is None or gpCurCore.is_empty(): break
        #    gpPrevCore = gpCurCore
        #    gpInputGraph = gpCurCore
        #    i+=1
        #    print("try ",i)
        #k = i-1
        #print("Answer ", lb)
        if self.l_cache:
            self.cache_kcore.update({x:lb for x in S_md5})
        return {'k':lb, 'network':gpPrevCore}
        #in the last iteration, gpCurCore is null (loop termination condition)

    #@staticmethod
    def calc_node_info(self, node, degreeCutoff=None):
        k = self.degree(node)
        neighbors = self.neighbors(node)
        s_md5=""
        #print("::::",node, ":::", k, "::::")
        #sw=util.StopWatch()
        if (k < 2):
            nodeInfo = NodeInfo()
            if (k == 1):
                nodeInfo.coreLevel = 1
                nodeInfo.coreDensity = 1.0
                nodeInfo.density = 1.0
                nodeInfo.numNodeNeighbors = len(neighbors); #########
                nodeInfo.nodeNeighbors = neighbors; ####
                # why ignore neighbor when k==1 in the original code???
        else:
            gpNodeNeighborhood = self.subnetwork(neighbors+[node])
            #sw.check('subnetwork')
            if (gpNodeNeighborhood.is_empty()):
                util.error_msg("In calc_node_info(): gpNodeNeighborhood was None.")
            #calculate the node information for each node
            if self.l_cache:
                s_md5=gpNodeNeighborhood.node_MD5()
                if s_md5 in self.cache_info:
                    #self.hit+=1
                    nodeInfo=self.cache_info[s_md5].clone()
                    nodeInfo.nodeNeighbors=neighbors
                    return nodeInfo
            nodeInfo = NodeInfo()
            #density
            nodeInfo.density = MCODE.calc_density(gpNodeNeighborhood, self.params['includeLoops'])
            #w.check('density')
            nodeInfo.numNodeNeighbors = len(neighbors)
            #calculate the highest k-core
            c = self.get_highest_KCore(gpNodeNeighborhood)
            #w.check('kcore')
            k = c['k']
            gpCore = c['network']
            nodeInfo.coreLevel = k
            if (gpCore is not None and not gpCore.is_empty()):
                nodeInfo.coreDensity = MCODE.calc_density(gpCore, self.params['includeLoops'])
            #w.check('cacl_density')
            #record neighbor array for later use in cluster detection step
            nodeInfo.nodeNeighbors = neighbors
        if degreeCutoff: nodeInfo.score_node(degreeCutoff)
        if self.l_cache:
            self.cache_info[s_md5]=nodeInfo
        return nodeInfo

    def score_graph(self):
        self.C_nodeInfo = {}
        self.C_nodeByScore = []
        S_node = self.nodes()
        rows=[]
        #sw=util.StopWatch()
        if self.CPU<=1:
            for node in S_node:
                nodeInfo = self.calc_node_info(node, self.params['degreeCutoff'])
                self.C_nodeInfo[node]=nodeInfo
                rows.append({'Node':node, 'Score':nodeInfo.score, 'Density':nodeInfo.density, 'numNodeNeighbors':nodeInfo.numNodeNeighbors})
        else:
            def f(X):
                return self.calc_node_info(X[0], X[1])
            #mp=parallel.MP()
            #mp.start(f, n_CPU=self.CPU)
            L=[ (x, self.params['degreeCutoff']) for x in S_node]
            out=parallel.parmap(f, L, n_CPU=self.CPU)
            #out=mp.map(L)
            for i,node in enumerate(S_node):
                self.C_nodeInfo[node]=out[i]
                rows.append({'Node':node, 'Score':out[i].score, 'Density':out[i].density, 'numNodeNeighbors':out[i].numNodeNeighbors})
        #sw.check('Done scoring')
        t=pd.DataFrame(rows)
        t=t.sort_values(['Score', 'Density', 'numNodeNeighbors', 'Node'], ascending=[False, False, False, True])
        grps=t.groupby(by='Score')
        self.C_nodeByScore=[ (score,list(grp['Node'])) for score,grp in grps]
        self.C_nodeByScore.sort(key=lambda x: x[0])
        self.C_nodeByScore.reverse()

    def get_cluster_core_internal(self, startNode, c_nodeSeen, startNodeScore, currentDepth, myCluster, nodeScoreCutoff, maxDepthFromStart):
        #base cases for recursion
        if (startNode in c_nodeSeen): return
        c_nodeSeen[startNode]=True
        if (currentDepth > maxDepthFromStart): return
        #don't exceed given depth from start node

        #Initialization
        neighbors = self.C_nodeInfo[startNode].nodeNeighbors
        #neighbors.sort()
        #print "A:"+startNode
        #print c_nodeSeen
        for node in neighbors:
            #go through all currentNode neighbors to check their core density for cluster inclusion
            #print "Neigh:"+node
            if (node in c_nodeSeen): continue
            if (self.C_nodeInfo[node].score >= (startNodeScore - startNodeScore * nodeScoreCutoff)):
                myCluster.append(node)
                #try to extend cluster at this node
                self.get_cluster_core_internal(node, c_nodeSeen, startNodeScore, currentDepth + 1, myCluster, nodeScoreCutoff, maxDepthFromStart)

    def get_cluster_core(self, startNode, c_nodeSeen, nodeScoreCutoff, maxDepthFromStart):
        myCluster = []
        self.get_cluster_core_internal(startNode, c_nodeSeen, self.C_nodeInfo[startNode].score, 1, myCluster, nodeScoreCutoff, maxDepthFromStart)
        return self.subnetwork(myCluster+[startNode])

    def fluff_cluster_boundary(self, myCluster, c_nodeSeen):
        #create a temp list of nodes to add to avoid concurrently modifying 'cluster'
        nodesToAdd = []
        #Keep a separate internal nodeSeenHashMap because nodes seen during a fluffing should not be marked as permanently seen,
        #they can be included in another cluster's fluffing step.
        c_nodeSeenInternal = {}
        #add all current neighbour's neighbours into cluster (if they have high enough clustering coefficients) and mark them all as seen
        S_node=myCluster.nodes()
        for node in S_node:
            neighbors = self.C_nodeInfo[node].nodeNeighbors
            for nb in neighbors:
                if (nb in c_nodeSeen): continue
                if (nb in c_nodeSeenInternal): continue
                if (self.C_nodeInfo[nb].density > self.params['fluffNodeDensityCutoff']):
                    nodesToAdd.append(nb)
                    c_nodeSeenInternal[nb]=True

        #Add fluffed nodes to cluster
        if (len(nodesToAdd)>0):
            return self.subnetwork(S_node+nodesToAdd)
        return myCluster

    def filter_cluster(self, gpClusterGraph):
        if (gpClusterGraph.is_empty()): return True
        #filter if the cluster does not satisfy the user specified k-core
        gpCore = MCODE.get_KCore(gpClusterGraph, self.params['kCore'])
        if (gpCore is None or gpCore.is_empty()): return True
        return False

    @staticmethod
    def haircut_cluster(myCluster):
        #get 2-core
        gpCore = MCODE.get_KCore(myCluster, 2)
        if (gpCore is not None or not gpCore.is_empty()):
            #clear the cluster and add all 2-core nodes back into it
            #must add back the nodes in a way that preserves gpInputGraph node indices
            # we cannot do myCluster=gpCore, which will not change myCluster outside
            return gpCore
        return myCluster

    def find_clusters(self, l_decompose=True, l_optimized=True):
        if (self.is_empty()):
            util.error_msg("In find_Clusters(): input network is empty!")
        if (not len(self.C_nodeInfo.keys()) or not len(self.C_nodeByScore)):
            util.error_msg("In find_Clusters(): C_nodeInfo or C_nodeByScore is None.")
        C_results=[]
        cnt=0
        #initialization
        c_nodeSeen= {} #key is nodeIndex, value is true/false
        c_nodeSeenSnapshot={}
        findingTotal = len(self.C_nodeInfo.keys())
        rows=[]
        for score,alNodesWithSameScore in self.C_nodeByScore:
            if not l_optimized or len(alNodesWithSameScore)<=1:
                for currentNode in alNodesWithSameScore:
                    if currentNode in c_nodeSeen: continue
                    alCluster = self.get_cluster_core(currentNode, c_nodeSeen, self.params['nodeScoreCutoff'], self.params['maxDepthFromStart'])
                    if (alCluster is not None and not alCluster.is_empty()):
                        #make sure seed node is part of cluster, if not already in there
                        if (not self.filter_cluster(alCluster)):
                            if (self.params['haircut']): alCluster=MCODE.haircut_cluster(alCluster)
                            if (self.params['fluff']): alCluster=self.fluff_Cluster_boundary(alCluster, c_nodeSeen)
                            if l_decompose:
                                c_components=alCluster.decompose()
                            else:
                                c_components=[alCluster]
                            for comp in c_components:
                                cnt+=1
                                score=self.score_network(comp)
                                C_results.append(MCODECluster(comp, currentNode, score))
                                rows.append({'ID':cnt, 'Score':score, 'NofNode':comp.nof_nodes(), 'SeedScore':self.C_nodeInfo[currentNode].score})
            else:
                def f(X):
                    tmp_rows=[]
                    c_stack={}
                    currentNode=X[0]
                    c_nodeSeenCopy=X[1].copy()
                    #if currentNode in c_nodeSeen: continue
                    alCluster = self.get_cluster_core(currentNode, c_nodeSeenCopy, self.params['nodeScoreCutoff'], self.params['maxDepthFromStart'])
                    if (alCluster is not None and not alCluster.is_empty()):
                        #make sure seed node is part of cluster, if not already in there
                        if (not self.filter_cluster(alCluster)):
                            if (self.params['haircut']): alCluster=MCODE.haircut_cluster(alCluster)
                            if (self.params['fluff']): alCluster=self.fluff_Cluster_boundary(alCluster, c_nodeSeenCopy)
                            if l_decompose:
                                c_components=alCluster.decompose()
                            else:
                                c_components=[alCluster]
                            for k,comp in enumerate(c_components):
                                score=self.score_network(comp)
                                tmp_rows.append({'ID':currentNode, 'Score':score, 'NofNode':comp.nof_nodes(), 'SeedScore':self.C_nodeInfo[currentNode].score, 'ComponentIndex':k})
                                c_stack[currentNode]={'nodeSeen':c_nodeSeenCopy, 'components':c_components}
                    return (tmp_rows, c_stack)

                while (len(alNodesWithSameScore)):
                    tmp_rows=[]
                    c_stack={}
                    L=[ (x, c_nodeSeen) for x in alNodesWithSameScore if x not in c_nodeSeen ]
                    #if self.CPU<=1:
                    #    out=[f(x) for x in L]
                    #else:
                    #    mp=parallel.MP()
                    #    mp.start(f, n_CPU=self.CPU)
                    #    out=mp.map(L)
                    out=parallel.parmap(f, L, n_CPU=self.CPU)
                    for X in out:
                        tmp_rows.extend(X[0])
                        c_stack.update(X[1])
                    tmp=pd.DataFrame(tmp_rows)
                    if len(tmp):
                        tmp=tmp.sort_values(['Score','NofNode','SeedScore', 'ID'], ascending=[False, False, False, True])
                        bestNode=tmp['ID'].iloc[0]
                        c_nodeSeen=c_stack[bestNode]['nodeSeen']
                        for comp in tmp_rows:
                            if comp['ID']!=bestNode: continue
                            compIdx=comp['ComponentIndex']
                            cnt+=1
                            C_results.append(MCODECluster(c_stack[bestNode]['components'][compIdx], bestNode, comp['Score']))
                            rows.append({'ID':cnt, 'Score':comp['Score'], 'NofNode':comp['NofNode'], 'SeedScore':self.C_nodeInfo[bestNode].score})

                        alNodesWithSameScore=[ x for x in alNodesWithSameScore if x !=bestNode]
                    else:
                        for x in c_stack:
                            for s in x.nodeSeen.keys():
                                c_nodeSeen[s]=True
                        alNodesWithSameScore=[]
        C_sorted=[]
        t=pd.DataFrame(rows)
        if len(t):
            t=t.sort_values(['Score','NofNode','SeedScore'], ascending=[False, False, False])
            for i in range(len(t)):
                C_sorted.append(C_results[t['ID'].iloc[i]-1])
        return C_sorted

    def to_MCODE_table(self, S_mcode_clusters):
        rows=[]
        for i,c in enumerate(S_mcode_clusters):
            S_nodes=c.nodes()
            for node in S_nodes:
                ty='Seed' if node==c.seedNode else 'Clustered'
                rows.append({'Cluster':i+1, 'Score':c.score, 'Type':ty, 'Gene':node})
        if len(rows)==0:
            return None
        t=pd.DataFrame(rows)
        if 'Symbol' in self.T_node.header():
            c_name={self.T_node['Gene'].iloc[i] : self.T_node['Symbol'].iloc[i] for i in range(len(self.T_node))}
            t['Symbol']=t['Gene'].map(c_name)
        return t

    @staticmethod
    def MCODE_label(network, s_col_name='MCODE_LABEL'):
        """Label nodes in the network by their MCODE cluster IDs, great for coloring nodes"""
        network=Network(network) # makes a copy
        L=network.decompose()
        c_attr={}
        for j,net in enumerate(L):
            mc=MCODE(net)
            mc.params['hariCut']=True
            components=mc.find_clusters(True, True)
            for i,c in enumerate(components):
                S_nodes=c.nodes()
                for x in S_nodes:
                    if x not in c_attr:
                        c_attr[x]="N%dC%d" % (j+1, i+1)
                    else:
                        c_attr[x]+=" N%dC%d" % (j+1, i+1)
        network.add_a_node_attr(s_col_name, c_attr)
        return network

    def __init__(self, network, n_CPU=0, l_cache=True):
        self.C_nodeInfo = None
        #key is the node name, value is a NodeInfo instance
        C_nodeByScore = None
        #a collection of array, {{Na, Nb}, {Nc}, {Nd,Ne} ...}, where nodes are sorted by descending score
        # nodes with the same NodeInfo.score are group in one array
        super(MCODE, self).__init__(network)
        #"Scoring all nodes in the network ..."
        self.CPU=n_CPU
        self.l_cache=l_cache
        #self.hit=0
        self.cache_info={}
        self.cache_kcore={}
        self.score_graph()
        #for c,v in self.C_nodeInfo.items():
        #    print c, v

class Cache(object):
    DATA_DIR=setting.ppi['DATA_DIR']
    ppi_data={'LOCAL':{}, 'GPDB':{}, 'HISTORY':{}}
    ppi_node={'LOCAL':{}, 'GPDB':{}, 'HISTORY':{}}
    ppi_edge={'LOCAL':{}, 'GPDB':{}, 'HISTORY':{}}
    CUTOFF_PHYS=132
    CUTOFF_COMB=187
    VERSION=setting.ppi.get('VERSION',2) # 1: without STRING DB, 2: with STRING DB, database scheme changed

    @staticmethod
    def gene2node(S_gene, con=None):
        if con is None: con=db.DB('METASCAPE')
        t_node=con.sql_in("SELECT gid Gene,source_id Symbol from gid2source_id t where gid in (", ") and t.id_type_id=1", util.rarray2iarray(S_gene))
        t_node['Gene']=t_node.Gene.astype(str)
        if len(S_gene)!=len(t_node):
            util.warn_msg("Strange, gene ID has no symbol?")
            t=pd.DataFrame({'Gene':list(S_gene)})
            t_node=t.merge(t_node, left_on='Gene', right_on='Gene', how='left')
            X=t_node.Symbol.isnull()
            #print(t_node.loc[X][:10])
            if X.any():
                t_node.loc[X,'Symbol']=t_node.loc[X,'Gene']
        return t_node

    @staticmethod
    def df2data(t, con=None):
        nodes=set(t.Gene_A)|set(t.Gene_B)
        data={ k:{} for k in nodes }
        [ (data[k].__setitem__(v,c) or data[v].__setitem__(k,c)) for k,v,c in zip(t.Gene_A, t.Gene_B, t.SCORE) ]
        return (data, Cache.gene2node(nodes, con=con))

    @staticmethod
    def get(l_use_GPDB=True, S_DB=None, tax_id=9606):
        """In VERSION=2, S_DB is a string, one of "PHYSICAL_CORE","PHYSICAL_ALL","COMBINED_CORE","COMBINED_ALL"
        getting a phyiscal db will populate both PHYSICAL_CORE and PHYSICAL_ALL
        getting a combined db will populate all four databases
        """
        S_DB=S_DB or Cache.get_DB(l_use_GPDB)
        if Cache.VERSION==1: # in version one we merge all db data in S_DB
            S_DB.sort()
            s_db=":".join(S_DB)
            if not (tax_id in Cache.ppi_data['HISTORY'] and s_db in Cache.ppi_data['HISTORY'][tax_id]):
                s_key=Cache.key(l_use_GPDB)
                Cache.load(tax_id=tax_id, l_use_GPDB=l_use_GPDB, S_DB=S_DB)
                data=None
                out_node=[]
                for x in S_DB:
                    #print ">>>>>>>>>", S_DB, x, Cache.ppi_data[s_key][tax_id].keys()
                    c=Cache.ppi_data[s_key][tax_id].get(x, {})
                    if data is None:
                        data=c
                    else:
                        for k in c.keys():
                            for v,score in c[k].items():
                                if k not in data:
                                    data[k]=c[k].copy()
                                else:
                                    data[k][v]=max(score, data[k].get(v,0))
                    out_node.append(Cache.ppi_node[s_key][tax_id].get(x, pd.DataFrame()))
                t_node=pd.concat(out_node, ignore_index=True)
                t_node.drop_duplicates('Gene', inplace=True)
                if tax_id not in Cache.ppi_data['HISTORY']:
                    Cache.ppi_data['HISTORY'][tax_id]={}
                    Cache.ppi_node['HISTORY'][tax_id]={}
                    Cache.ppi_edge['HISTORY'][tax_id]={}
                Cache.ppi_data['HISTORY'][tax_id][s_db]=data
                Cache.ppi_node['HISTORY'][tax_id][s_db]=t_node
        else: # In VERSION 2, each entry in S_DB is its own collection
            s_db=S_DB
            #print(tax_id, list(Cache.ppi_data['HISTORY'].keys()), list(Cache.ppi_data['HISTORY'][tax_id].keys()))
            if not (tax_id in Cache.ppi_data['HISTORY'] and s_db in Cache.ppi_data['HISTORY'][tax_id]):
                Cache.load(tax_id=tax_id, l_use_GPDB=True, S_DB=s_db)
        return (Cache.ppi_data['HISTORY'][tax_id][s_db], Cache.ppi_node['HISTORY'][tax_id][s_db], \
                Cache.ppi_edge['HISTORY'][tax_id].get(s_db, None))

    @staticmethod
    def info():
        for s_key in ('LOCAL','GPDB','HISTORY'):
            print(">Databases: %s" % s_key)
            for tax_id in Cache.ppi_data[s_key].keys():
                print("TAX_ID=%d (%s)" % (tax_id, ez.Cache.C_TAX_NAME.get(tax_id, "UNKNOWN")))
                for s_db in Cache.ppi_data[s_key][tax_id].keys():
                    print("Source: %s" % s_db)
                    print("PPI_DATA=%d" % len(Cache.ppi_data[s_key][tax_id][s_db]))
                    print("PPI_NODE=%d" % len(Cache.ppi_node[s_key][tax_id][s_db]))
                    print("PPI_EDGE=%d" % len(Cache.ppi_edge[s_key][tax_id][s_db]))
            print("")

    @staticmethod
    def unload(tax_id, l_use_GPDB):
        s_key=Cache.key(l_use_GPDB)
        if tax_id in Cache.ppi_data[s_key]:
            del Cache.ppi_data[s_key][tax_id]
            del Cache.ppi_node[s_key][tax_id]

    @staticmethod
    def key(l_use_GPDB):
        return 'GPDB' if l_use_GPDB else 'LOCAL'

    @staticmethod
    def get_DB(l_use_GPDB=True):
        if Cache.VERSION==1:
            DEFAULT_DB=["BioGrid","InWeb_IM","OmniPath"] if l_use_GPDB else ["BHMRRS","CORUM","Prolexys","Chanda"] # String
        else:
            DEFAULT_DB=setting.ppi.get('DEFAULT_DB', ["PHYSICAL_CORE","PHYSICAL_ALL","COMBINED_CORE","COMBINED_ALL"][2])
        return DEFAULT_DB

    @staticmethod
    def load(tax_id=9606, l_use_GPDB=True, S_DB=None, entrez=None):
        """tax_id is None, defaults to 9606, if 0, means load all supported species,
        entrez is only used in local mode to accelerate Symbol retrieval"""
        sw=util.StopWatch()
        if Cache.VERSION==2:
            if S_DB is None: S_DB="PHYSICAL_CORE"
            if type(S_DB)!=str: util.error_msg("S_DB must be a string in VERSION 2")
            s_db=S_DB
            fn=setting.ppi.get('STRING_PATH', os.path.join(os.path.dirname(__file__),"STRING/Interaction.csv.gz"))
            mydb=db.DB('METASCAPE')
            if tax_id==0:
                S_tax_id=ez.Cache.C_TAX_ID.values()
            else:
                S_tax_id=[tax_id]
            data=[]
            for i_tax_id in S_tax_id:
                fn=setting.ppi.get('STRING_PATH', os.path.join(os.path.dirname(__file__), f"STRING/Interaction.{i_tax_id}.csv.gz"))
                if os.path.exists(fn):
                    t=util.read_csv(fn, dtype={'gid_A':str, 'gid_B':str})
                    if "PHYSICAL" in s_db:
                        t=t[t.interaction_type_id==11].copy()
                    t.rename2({'gid_A':'Gene_A', 'gid_B':'Gene_B', 'tax_id_A':'tax_id'})
                    sw.check(f"data loaded from {fn}")
                else:
                    if i_tax_id>0:
                        if "PHYSICAL" in s_db:
                            t=mydb.from_sql("SELECT gid_A Gene_A,gid_B Gene_B,interaction_type_id,score_physical,score_combined,tax_id_A tax_id,support from interaction where tax_id_A=? and interaction_type_id=11", params=[i_tax_id])
                        else:
                            t=mydb.from_sql("SELECT gid_A Gene_A,gid_B Gene_B,interaction_type_id,score_physical,score_combined,tax_id_A tax_id,support from interaction where tax_id_A=?", params=[i_tax_id])
                    else:
                        if "PHYSICAL" in s_db:
                            t=mydb.from_sql("SELECT gid_A Gene_A,gid_B Gene_B,interaction_type_id,score_physical,score_combined,tax_id_A tax_id,support from interaction where interaction_type_id=11")
                        else:
                            t=mydb.from_sql("SELECT gid_A Gene_A,gid_B Gene_B,interaction_type_id,score_physical,score_combined,tax_id_A tax_id,support from interaction")
                    t['Gene_A']=t.Gene_A.astype(str)
                    t['Gene_B']=t.Gene_B.astype(str)
                if sum(t.Gene_A>t.Gene_B):
                    util.info_msg("Genes not order by str, canonicalize required!")
                    t=Network.canonicalize_table(t) # since we change type to str, we need to reorder it
                data.append(t)
            if len(data)==1:
                t=data[0]
            else:
                t=pd.concat(data, ignore_index=True)
            #sw.check("Canonicalized")
            t['TYPE']='Direct'
            sw.check("Start processing each tax_id")
            S_tax_id=t.tax_id.unique()
            for tax_id in S_tax_id:
                #for tax_id,t_v in t.groupby('tax_id'):
                #sw.check("ENTER GROUPBY")
                if tax_id not in Cache.ppi_data['HISTORY']:
                    Cache.ppi_data['HISTORY'][tax_id]={}
                    Cache.ppi_node['HISTORY'][tax_id]={}
                    Cache.ppi_edge['HISTORY'][tax_id]={}
                if "COMBINED" in s_db:
                    tmp=t.loc[t.tax_id==tax_id, ['Gene_A','Gene_B','TYPE','score_combined','support']].copy()
                    #sw.check("COPY")
                    tmp.rename2({'score_combined':'SCORE'})
                    data,t_node=Cache.df2data(tmp, con=mydb)
                    #sw.check("DICT")
                    Cache.ppi_data['HISTORY'][tax_id]["COMBINED_ALL"]=data
                    Cache.ppi_node['HISTORY'][tax_id]["COMBINED_ALL"]=t_node
                    Cache.ppi_edge['HISTORY'][tax_id]["COMBINED_ALL"]=tmp
                    #sw.check("Combined all")
                    tmp=tmp[tmp.SCORE>=Cache.CUTOFF_COMB].copy()
                    #sw.check("FILTER")
                    data,t_node=Cache.df2data(tmp, con=mydb)
                    #sw.check("DICT2")
                    Cache.ppi_data['HISTORY'][tax_id]["COMBINED_CORE"]=data
                    Cache.ppi_node['HISTORY'][tax_id]["COMBINED_CORE"]=t_node
                    Cache.ppi_edge['HISTORY'][tax_id]["COMBINED_CORE"]=tmp
                    #tmp=t_v[t_v.interaction_type_id==11]
                tmp=t.loc[(t.tax_id==tax_id) & (t.interaction_type_id==11)]
                tmp=tmp[['Gene_A','Gene_B','TYPE','score_physical','support']].copy()
                tmp.rename2({'score_physical':'SCORE'})
                #sw.check("Combined core")
                data,t_node=Cache.df2data(tmp, con=mydb)
                Cache.ppi_data['HISTORY'][tax_id]["PHYSICAL_ALL"]=data
                Cache.ppi_node['HISTORY'][tax_id]["PHYSICAL_ALL"]=t_node
                Cache.ppi_edge['HISTORY'][tax_id]["PHYSICAL_ALL"]=tmp
                #sw.check("Physical all")
                tmp=tmp[tmp.SCORE>=Cache.CUTOFF_COMB].copy()
                data,t_node=Cache.df2data(tmp, con=mydb)
                Cache.ppi_data['HISTORY'][tax_id]["PHYSICAL_CORE"]=data
                Cache.ppi_node['HISTORY'][tax_id]["PHYSICAL_CORE"]=t_node
                Cache.ppi_edge['HISTORY'][tax_id]["PHYSICAL_CORE"]=tmp
                #sw.check("Physical core")
                sw.check(f"processed :{tax_id}")
                t=t.loc[t.tax_id!=tax_id]
            return

        S_DB=S_DB or Cache.get_DB(l_use_GPDB)
        if tax_id is None:
            util.error_msg('tax_id must be an int, or 0 means all supported species')
        tax_id=abs(tax_id)
        s_key=Cache.key(l_use_GPDB)
        S_tax_id=[]
        if not l_use_GPDB:
            if tax_id not in (0,9606):
                util.error_msg('Local database only supports human!')
            tax_id=9606
            if tax_id in Cache.ppi_data[s_key]:
                S_DB=[x for x in S_DB if x not in Cache.ppi_data[s_key][tax_id]]
                if len(S_DB)==0: return
            S_tax_id=[tax_id]
            T=[]
            for filename in S_DB:
                print("loading PPI database: "+filename+" ...")
                if os.path.isfile(filename):
                    t=pd.read_csv(filename)
                    t['ds']=filename
                    T.append(t)
                elif os.path.isfile(Cache.DATA_DIR+filename+".csv"):
                    t=pd.read_csv(Cache.DATA_DIR+filename+".csv")
                    t['ds']=filename
                    T.append(t)
                else:
                    util.warn_msg('PPI database ' + filename + ' not found.')
            if len(T)>1:
                t=pd.concat(T, axis=0, ignore_index=True)
            else:
                t=T[0]
            t=t[(t.Gene_A!=t.Gene_B) & (t.Score>=0.5)].copy()
            eg=entrez
            if eg is None:
                eg=ez.EntrezGene(tax_id=tax_id)
            else:
                eg.load_organism(tax_id=tax_id)
            c_seen={}
            t.index=list(range(len(t)))
            t['Gene_A']=t.Gene_A.astype(str)
            t['Gene_B']=t.Gene_B.astype(str)
            S_gene_A=t.Gene_A.tolist()
            S_gene_B=t.Gene_B.tolist()
            for i in range(len(t)):
                gene_A=S_gene_A[i]
                gene_B=S_gene_B[i]
                if gene_A not in c_seen:
                    c_seen[gene_A]=eg.fix_gene_id(gene_A)
                S_gene_A[i]=c_seen[gene_A]
                if S_gene_A[i] is None: continue
                if gene_B not in c_seen:
                    c_seen[gene_B]=eg.fix_gene_id(gene_B)
                S_gene_B[i]=c_seen[gene_B]
            t['Gene_A']=S_gene_A
            t['Gene_B']=S_gene_B
            t=t[~(t.Gene_A.isnull() | t.Gene_B.isnull())].copy()
            t.index=list(range(len(t)))
            t['tax_id']=tax_id
        else:
            mydb=db.DB('METASCAPE')
            if tax_id>0 and tax_id in Cache.ppi_data[s_key]:
                S_DB=[x for x in S_DB if x not in Cache.ppi_data[s_key][tax_id]]
                if len(S_DB)==0: return
            if tax_id>0:
                print("loading PPI database from database for tax_id: %d ..." % tax_id)
                t=mydb.sql_in("SELECT gid_A Gene_A,gid_B Gene_B,0 Score,tax_id_A tax_id,ds from interaction where interaction_category!='genetic' and gid_A!=gid_B and tax_id_A=tax_id_B and tax_id_A=? and ds in (", ")", S_DB, params_before=[tax_id])
                S_tax_id=[tax_id]
            else:
                #ZZZ modify in the future, to obtain the list of all supported tax_id
                t=mydb.from_sql('SELECT DISTINCT tax_id FROM gid2source_id')
                S_tax_id=[x for x in t.tax_id.astype(int).tolist() if x not in Cache.ppi_data[s_key]]
                if len(S_tax_id):
                    s_tax_id=",".join(util.iarray2sarray(S_tax_id))
                    print("loading PPI database for tax_id: %s ..." % s_tax_id)
                    t=mydb.sql_in("SELECT gid_A Gene_A,gid_B Gene_B,0 Score,tax_id_A tax_id,ds from interaction where interaction_category!='genetic' and gid_A!=gid_B and tax_id_A=tax_id_B and ds in (", ")", S_DB)
                    #t=mydb.sql_in("SELECT gid_A Gene_A,gid_B Gene_B,0 Score,tax_id_A tax_id,ds from interaction where interaction_category!='genetic' and gid_A!=gid_B and tax_id_A=tax_id_B and tax_id_A in ("+s_tax_id+") and ds in (", ")", S_DB)
                else:
                    t=pd.DataFrame()
            if len(t):
                t['Gene_A']=t.Gene_A.astype(str)
                t['Gene_B']=t.Gene_B.astype(str)
                if sum(t.Gene_A>t.Gene_B):
                    t=Network.canonicalize_table(t) # since we change type to str, we need to reorder it

        for x in S_tax_id:
            #print ">>>>>>>>>>>>>>>>>>>>>>>", x
            if x not in Cache.ppi_data[s_key]:
                Cache.ppi_data[s_key][x]={}
                Cache.ppi_node[s_key][x]={}
            for y in S_DB:
                Cache.ppi_data[s_key][x][y]={}
                Cache.ppi_node[s_key][x][y]=pd.DataFrame()

        if len(t)==0: return
        for k,t_v in t.groupby(['tax_id','ds']):
            #print ">>>", k, len(t_v)
            #t_v=t_v.copy()
            if k[0] not in S_tax_id: continue
            data={}
            t_node=None
            #t_v=t_v.copy()
            #t_v.index=list(range(len(t_v)))
            #for i in t_v.index:
                #if i%1000==0: print i
            for row in t_v.itertuples():
                gene_A=row.Gene_A #t_v.ix[i,'Gene_A']
                gene_B=row.Gene_B #t_v.ix[i,'Gene_B']
                score=row.Score #t_v.ix[i,'Score']
                if gene_A not in data:
                    data[gene_A]={gene_B:score}
                else:
                    data[gene_A][gene_B]=max(score, data[gene_A].get(gene_B,0))
                if gene_B not in data:
                    data[gene_B]={gene_A:score}
                else:
                    data[gene_B][gene_A]=max(score, data[gene_B].get(gene_A,0))
            Cache.ppi_data[s_key][k[0]][k[1]]=data
            S_gene=list(data.keys())
            if l_use_GPDB:
                t_node=Cache.gene2node(S_gene, con=mydb)
            else:
                t_node=eg.gene_sarray_to_table(S_gene, l_description=False)
            Cache.ppi_node[s_key][k[0]][k[1]]=t_node

# YZHOU: for InWeb_IM, their web GUI uses a threshold for score
#From: Rasmus Borup Hansen [mailto:rbh@intomics.com]
#Sent: Friday, February 03, 2017 4:22 AM
#Subject: Re: Interaction not shown in InBio Map
#
#To make a long story short: We've tried a number of different strategies for choosing a cutoff, and right now the web interface uses 0.156.
#
#Best,
#
#Rasmus

class PPI(Network):

    def __init__(self, tax_id=9606, l_use_GPDB=False, S_DB=None):
        """tax_id is None, defaults to 9606, if 0, means load all species
        Warning: S_DB is set in Cache.load(), so preload Cache if you want to use different database"""
        self.tax_id=tax_id
        data, t_node, t_edge=Cache.get(tax_id=tax_id, l_use_GPDB=l_use_GPDB, S_DB=S_DB)
        print("PPI databases loaded")
        super(PPI, self).__init__(data, T_node=t_node, name='proteome', premade_T_edge=t_edge, skip_copy=True)

if __name__=="__main__":

    #Cache.load(tax_id=9606, S_DB='PHYSICAL_CORE')
    #Cache.load(tax_id=9606, S_DB='COMBINED_CORE')
    sw=util.StopWatch()
    Cache.load(tax_id=9606, S_DB='COMBINED_CORE')
    Cache.info()
    sw.check('Loaded')
    #Cache.load(tax_id=0, l_use_GPDB=True)
    #Cache.load(tax_id=0, S_DB=['BioGrid','GeneGO'], l_use_GPDB=True)
    #Cache.info()
    #exit()
    ppi=PPI(l_use_GPDB=True, tax_id=9606)
    sw.check('Ready')
    exit()
    ppi=PPI(l_use_GPDB=True, tax_id=9606)
    print(list(Cache.ppi_data['GPDB'].keys()))
    #ppi.T_node.to_csv('t1.csv')
    #ppi.T_edge.to_csv('t2.csv')
    print(ppi.data['132884'])
    S_node=['132884','191','537']
    test=ppi.subnetwork(S_node)
    print(test.nof_nodes())
    exit()

    ## example
    S_node=util.read_list('~/RM_Hits.txt')
    test=ppi.subnetwork(S_node)
    test.to_xgmml('RM_.xgmml')
    exit()
    S_node=util.read_list('~/CM_Hits.txt')
    test=ppi.subnetwork(S_node)
    test.to_xgmml('CM_.xgmml')

    exit()
    #print ppi.T_node[:5]
    #print ppi.T_edge[:5]
    test=ppi.subnetwork(S_node)
    #print test
    exit()

    mc=MCODE(net)
    #print mc.C_nodeByScore
    mc.params['hairCut']=True
    c=mc.find_clusters(True, True)
    print(mc.to_MCODE_table(c))
    for i,cp in enumerate(c):
        print(">>> Rank "+str(i)+" <<<")
        cp.to_xgmml('out/test'+str(i))
        S=cp.nodes()
        for node in S:
            nodeInfo=mc.C_nodeInfo[node]
            print("Node=> "+node)
            print(nodeInfo)


import Server
import os
import socket
import threading
from tkinter import *
from tkinter import filedialog, messagebox

import customtkinter as ctk
from CTkListbox import *

import PIL.Image
import PIL.ImageTk
from Server import *

SIZE = 1024
FORMAT = "utf-8"
PORT = 4000

connFlag = [False, 0]

def serverLogWindow(frame, main_window, folderpath):
    # l=[f for f in os.listdir(folderpath) if os.path.isfile(os.path.join(folderpath, f))]
    l = []

    IP = socket.gethostbyname(socket.gethostname() + ".local")
    ADDR = (IP, PORT)

    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sock.bind(ADDR)
    sock.listen()

    devicesConnectedVar = StringVar()
    devicesConnectedVar.set(f"Devices connected:{connFlag[1]}")

    serverLogs = []
    serverLogsVar = Variable(value=str(serverLogs))

    serverLogs.append("[STARTING] Server is starting...")
    serverLogsVar.set(str(serverLogs))

    print("Waiting for connection...")
    print(f"Share this code: {IP}")

    def checkChanges(server, addr):
        l = []
        while True:
            new_l = [
                f
                for f in os.listdir(folderpath)
                if os.path.isfile(os.path.join(folderpath, f))
            ]
            if connFlag[0] == True or l != new_l:
                l = new_l
                msg = f"UPDATE%{str(new_l)}"
                # print(msg)
                server.send(convertToSIZE(msg))
                connFlag[0] = False

    def serverHandler():
        while True:
            conn, addr = sock.accept()

            if conn:
                serverConnected(conn, addr)

    def serverConnected(conn, addr):
        global server

        server = conn
        msg = "[NEW CLIENT] : HOST"
        # server.send(convertToSIZE(msg.encode(FORMAT)))
        server.send(msg.encode(FORMAT))

        serverLogs.append(f"[NEW CONNECTION]: {addr} Connected")
        serverLogsVar.set(str(serverLogs))

        clientThread = threading.Thread(target=clientHandlerHost, args=(server, addr))
        clientThread.start()

        connFlag[1] += 1
        devicesConnectedVar.set(f"Devices connected:{connFlag[1]}")

        # if connFlag[1] == 1:
        checkChangeThread = threading.Thread(target=checkChanges, args=(server, addr))
        checkChangeThread.start()

    def clientHandlerHost(server, addr):
        new_l = [
            f
            for f in os.listdir(folderpath)
            if os.path.isfile(os.path.join(folderpath, f))
        ]

        msg = f"UPDATE%{str(new_l)}"
        server.send(convertToSIZE(msg))

        while True:
            msg = server.recv(SIZE)
            if not msg:
                continue
            else:
                msg = removeExtraBytes(msg).decode(FORMAT)

                cmd, msg = msg.split("%")

                if cmd == "DOWNLOAD":
                    filename = msg

                    filepath = os.path.join(folderpath, filename)
                    packetCount = getPacketCount(filepath)

                    fileData = f'["{filename}", "{packetCount}"]'

                    msgSend = f"TAKE_DATA%{fileData}"
                    server.send(convertToSIZE(msgSend))

                    with open(filepath, "rb") as f:
                        while packetCount > 0:
                            data = f.read(SIZE)
                            server.send(data)
                            packetCount -= 1

                    serverLogs.append(f"[DOWNLOADED]: {filename} downloaded by {addr}")
                    serverLogsVar.set(str(serverLogs))

                elif cmd == "UPLOAD":
                    filename = msg
                    filepath = os.path.join(folderpath, filename).replace("\\", "/")

                    packetCountResponse = eval(
                        removeExtraBytes(server.recv(SIZE)).decode(FORMAT)
                    )

                    with open(filepath, "wb") as f:
                        while packetCountResponse > 0:
                            data = server.recv(SIZE)
                            f.write(data)
                            packetCountResponse -= 1

                    connFlag[0] = True

                    serverLogs.append(f"[UPLOADED]: {filename} uploaded by {addr}")
                    serverLogsVar.set(str(serverLogs))

                elif cmd == "DELETE":
                    filename = msg
                    filepath = os.path.join(folderpath, filename).replace("\\", "/")

                    os.remove(filepath)
                    connFlag[0] = True
                    serverLogs.append(f"[DELETED]: {filename} deleted by {addr}")
                    serverLogsVar.set(str(serverLogs))

    def shutDownServerButton():
        frame.destroy()
        main_window()

    serverThread = threading.Thread(target=serverHandler)
    serverThread.start()

    lb1 = ctk.CTkLabel(
        frame,
        text="SERVER LOGS",
        font=("Comic Sans MS bold", 20),
        padx=5,
        pady=5,
    )
    lb1.pack()

    lb2 = ctk.CTkLabel(
        frame,
        text=f"Share this code to join: {IP}",
        font=("Comic Sans MS bold", 18),
        padx=5,
        pady=5,
    )
    lb2.place(x=10, y=50)

    lb3 = ctk.CTkLabel(
        frame,
        textvariable=devicesConnectedVar,
        font=("Comic Sans MS bold", 18),
        padx=5,
        pady=5,
    )
    lb3.place(x=790, y=50)

    listbox1 = CTkListbox(frame, listvariable=serverLogsVar, height=400, width=950)
    listbox1.place(x=10, y=100)

    btn1 = ctk.CTkButton(frame, text="Shut Down Server", command=shutDownServerButton)
    btn1.place(x=450, y=550)


from jinja2 import Template
f = open('weather.log', 'r')
w = f.readlines()
f.close()

weather = []
for i in w:
    weather.append(i[:25].strip().split())

#print(weather)
html = open('weth.txt').read()
template = Template(html)
render = template.render(weather = weather)

f = open('weather.html', 'w')
f.write(render)
f.close()


import os
import sys

import numpy as np
import pytest
from numpy.random import normal

sys.path.append(os.path.join(os.path.dirname(__file__), "../.."))
module = __import__("Acquisition", fromlist=["EI"])

class TestEI:
    @pytest.mark.parametrize(
        ("mean", "var", "base"),
        [(0.0, 1.0, 0.5), (0.8, 0.1, 0.3), (0.1, 1.3, 0.7), (0.4, 0.1, 0.3),],
    )
    def test_f(self, mean, var, base) -> float:
        # 厳密解
        AF = module.EI.EI()
        ei = AF.f(mean, var, base)

        # 近似解(モンテカルロ積分)
        sum = 0.0
        size = 100000
        for x in normal(mean, var, size):
            sum += np.max([base - x, 0.0])
        sum /= size
        print(sum, ei)

        # 差分
        dif = np.abs(sum - ei)
        assert dif < 0.1


"""empty message

Revision ID: d21d3839c096
Revises: 0c6b29c57638
Create Date: 2020-01-08 12:54:10.048577

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = 'd21d3839c096'
down_revision = '0c6b29c57638'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('word_classification',
    sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
    sa.Column('word_classification', sa.String(length=4), nullable=False),
    sa.Column('word', sa.String(length=1024), nullable=False),
    sa.Column('last_name_used', sa.String(length=1024), nullable=True),
    sa.Column('last_prep_name', sa.String(length=1024), nullable=True),
    sa.Column('frequency', sa.BIGINT(), nullable=True),
    sa.Column('approved_by', sa.Integer(), nullable=True),
    sa.Column('approved_dt', sa.DateTime(timezone=True), nullable=True),
    sa.Column('start_dt', sa.DateTime(timezone=True), nullable=True),
    sa.Column('end_dt', sa.DateTime(timezone=True), nullable=True),
    sa.Column('last_updated_by', sa.Integer(), nullable=True),
    sa.Column('last_update_dt', sa.DateTime(timezone=True), nullable=True),
    sa.ForeignKeyConstraint(['approved_by'], ['users.id'], ),
    sa.ForeignKeyConstraint(['last_updated_by'], ['users.id'], ),
    sa.PrimaryKeyConstraint('id')
    )
    op.create_index(op.f('ix_word_classification_word'), 'word_classification', ['word'], unique=False)
    op.create_index(op.f('ix_word_classification_word_classification'), 'word_classification', ['word_classification'], unique=False)
   # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_index(op.f('ix_word_classification_word_classification'), table_name='word_classification')
    op.drop_index(op.f('ix_word_classification_word'), table_name='word_classification')
    op.drop_table('word_classification')
    # ### end Alembic commands ###


import sys
sys.path.insert(0,'python')
from geo_trans import *
import numpy as np
from scipy.interpolate import griddata
from fast_rw import *

def get_coords(h,v):
    mgrss = get_lon_lat(27, 5).ravel()

    mgrss = np.array([(i[:5],i[-8:-4],i[-4:]) for i in mgrss]).reshape(2400,2400,3)

    index = np.where(mgrss[:,:,0]=='50SMG')
    Scoords = [9999-mgrss[index[0], index[1],2].astype('int'), mgrss[index[0], index[1],1].astype('int')]
    return index, Scoords

h=27; v=5
Rs =  np.arange(2400*2400).reshape(2400,2400)
grid_x, grid_y = np.mgrid[0:10980, 0:10980]
index, Scoords = get_coords(h,v)

values = Rs[index[0],index[1]]
std_int_sent = griddata(np.array(zip(Scoords[0],Scoords[1])), values, (grid_x, grid_y), method='nearest')
parallel_rw_pkl(std_int_sent, 'std_int_sent%i', 'w')


__author__ = 'James DeVincentis <james.d@hexhost.net>'

import os
import multiprocessing
import time

import schedule
import setproctitle

import cif

class Feeder(multiprocessing.Process):
    def __init__(self):
        multiprocessing.Process.__init__(self)
        self.backend = None
        self.logging = cif.logging.getLogger('FEEDER')
        self.logging.info("Loading Feeds")
        self.load_feeds()

    def load_feeds(self):
        schedule.clear()
        feeds = {}
        self.logging.debug("Getting List of Feeds")
        files = os.listdir(cif.options.feed_directory)
        feed_files = []
        for file in files:
            if file.endswith(".yml"):
                self.logging.debug("Found Feed File: {0}".format(file))
                feed_files.append(os.path.join(cif.options.feed_directory, file))

        feed_files.sort()
        for feed_file in feed_files:
            self.logging.info("Loading Feed File: {0}".format(feed_file))
            feeds[feed_file] = cif.feeder.Feed(feed_file)
            self.logging.info("Scheduling Feed File: {0}".format(feed_file))
            if 'feeds' not in feeds[feed_file].feed_config:
                self.logging.info("{0} does not contain feeds key".format(feed_file))
                continue
            for feed_name in feeds[feed_file].feed_config['feeds'].keys():
                if "interval" in feeds[feed_file].feed_config['feeds'][feed_name]:
                    if feeds[feed_file].feed_config['feeds'][feed_name]['interval'] == "hourly":
                        self.logging.info(repr(schedule.every().hour.at("00:00").do(feeds[feed_file].process, feed_name)))
                    elif feeds[feed_file].feed_config['feeds'][feed_name]['interval'] == "daily":
                        self.logging.info(repr(schedule.every().day.at("00:00").do(feeds[feed_file].process, feed_name)))
                    elif feeds[feed_file].feed_config['feeds'][feed_name]['interval'] == "weekly":
                        self.logging.info(repr(schedule.every().day.at("00:00").do(feeds[feed_file].process, feed_name)))
                else:
                    self.logging.info(repr(schedule.every(1).minute.do(feeds[feed_file].process, feed_name)))

    def run(self):
        try:
            setproctitle.setproctitle('[CIF-SERVER] - Feeder')
        except:
            pass
        while True:
            try:
                schedule.run_pending()
                time.sleep(1)
            except Exception as e:
                self.logging.error("Schedule dead, restarting")
                continue


from typing import Any, Dict, List, Type, TypeVar, Union

import attr

from ..models.share_credential import ShareCredential
from ..types import UNSET, Unset

T = TypeVar("T", bound="ModifyRepositoryProfileRequest")

@attr.s(auto_attribs=True)
class ModifyRepositoryProfileRequest:
    """Model having repository details

    Attributes:
        profile_name (Union[Unset, str]): name of the repository profile
        description (Union[Unset, str]):
        share_path (Union[Unset, str]): Provide the share path of catalog
        modified_by (Union[Unset, str]): Provide the modifiedby user
        share_credential (Union[Unset, ShareCredential]): Share credential details
    """

    profile_name: Union[Unset, str] = UNSET
    description: Union[Unset, str] = UNSET
    share_path: Union[Unset, str] = UNSET
    modified_by: Union[Unset, str] = UNSET
    share_credential: Union[Unset, ShareCredential] = UNSET
    additional_properties: Dict[str, Any] = attr.ib(init=False, factory=dict)

    def to_dict(self) -> Dict[str, Any]:
        profile_name = self.profile_name
        description = self.description
        share_path = self.share_path
        modified_by = self.modified_by
        share_credential: Union[Unset, Dict[str, Any]] = UNSET
        if not isinstance(self.share_credential, Unset):
            share_credential = self.share_credential.to_dict()

        field_dict: Dict[str, Any] = {}
        field_dict.update(self.additional_properties)
        field_dict.update({})
        if profile_name is not UNSET:
            field_dict["profileName"] = profile_name
        if description is not UNSET:
            field_dict["description"] = description
        if share_path is not UNSET:
            field_dict["sharePath"] = share_path
        if modified_by is not UNSET:
            field_dict["modifiedBy"] = modified_by
        if share_credential is not UNSET:
            field_dict["shareCredential"] = share_credential

        return field_dict

    @classmethod
    def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T:
        d = src_dict.copy()
        profile_name = d.pop("profileName", UNSET)

        description = d.pop("description", UNSET)

        share_path = d.pop("sharePath", UNSET)

        modified_by = d.pop("modifiedBy", UNSET)

        _share_credential = d.pop("shareCredential", UNSET)
        share_credential: Union[Unset, ShareCredential]
        if isinstance(_share_credential, Unset):
            share_credential = UNSET
        else:
            share_credential = ShareCredential.from_dict(_share_credential)

        modify_repository_profile_request = cls(
            profile_name=profile_name,
            description=description,
            share_path=share_path,
            modified_by=modified_by,
            share_credential=share_credential,
        )

        modify_repository_profile_request.additional_properties = d
        return modify_repository_profile_request

    @property
    def additional_keys(self) -> List[str]:
        return list(self.additional_properties.keys())

    def __getitem__(self, key: str) -> Any:
        return self.additional_properties[key]

    def __setitem__(self, key: str, value: Any) -> None:
        self.additional_properties[key] = value

    def __delitem__(self, key: str) -> None:
        del self.additional_properties[key]

    def __contains__(self, key: str) -> bool:
        return key in self.additional_properties


from collections import deque

idx = 1
while True:
    N = int(input())
    if N == 0:
        exit()

    arr = [list(map(int, input().split())) for _ in range(N)]
    cost = [[999]*N for _ in range(N)]
    visited = [[False]*N for _ in range(N)]

    q = deque([[0,0]])
    cost[0][0] = arr[0][0]
    dx = [-1, 1, 0, 0] # 상하좌우
    dy = [0, 0, -1, 1]

    while q:
        x, y = q.popleft()
        visited[0][0] = True
        for i in range(4):
            nx = x + dx[i]
            ny = y + dy[i]
            if 0 <= nx < N and 0 <= ny < N and not visited[nx][ny]:
                if cost[nx][ny] > cost[x][y] + arr[nx][ny]:
                    cost[nx][ny] = cost[x][y] + arr[nx][ny]
                    q.append([nx,ny])

    print(f'Problem {idx}: {cost[N-1][N-1]}')
    idx += 1


import pyaudio
import numpy as np
import math
import struct
import simpleaudio as sa
from imutils.video import FPS

#CHUNK = 1024
CHUNK = 1024
FORMAT = pyaudio.paInt16
CHANNELS = 2
RATE = 44100
RECORD_SECONDS = 0.03
print(int(RATE / CHUNK * RECORD_SECONDS))

volume = 0.1     # range [0.0, 1.0]
fs = 44100       # sampling rate, Hz, must be integer
duration = 0.3   # in seconds, may be float
f = 750.0        # sine frequency, Hz, may be float

fps = FPS().start()

def rms(data):
    if data != []:
        count = len(data)/2
        format = "%dh"%(count)
        shorts = struct.unpack(format, data)
        sum_squares = 0.0
        for sample in shorts:
            n = sample * (1.0/32768)
            sum_squares += n*n
        return math.sqrt(sum_squares / count)
    return 0

p = pyaudio.PyAudio()
p2 = pyaudio.PyAudio()

stream = p.open(format=FORMAT,
                channels=CHANNELS,
                rate=RATE,
                input=True,
                frames_per_buffer=CHUNK)

# for paFloat32 sample values must be in range [-1.0, 1.0]
streamout = p2.open(format=pyaudio.paFloat32,
                   channels=1,
                   rate=fs,
                   output=True)

# generate samples, note conversion to float32 array
samples = (np.sin(2*np.pi*np.arange(fs*duration)*f/fs)).astype(np.float32)

wave_obj = sa.WaveObject.from_wave_file("bumbo.wav")
ind = 0
count = 0
while ind != 500:
    frames = []
    energy = []
    for _ in range(0, int(RATE / CHUNK * RECORD_SECONDS)):
        try:
            data = stream.read(CHUNK, exception_on_overflow=False)
        except IOError:
            #data = '\x00' * CHUNK
            data = []
        energy.append(rms(data))
    if np.mean(energy) > 0.01 and count > 10:
        #wave_obj.play()
        streamout.get_write_available()
        #streamout.write(samples * np.mean(energy)/0.2)
        streamout.write(samples * (volume + np.mean(energy) * 5))
        print(np.mean(energy))
        # energy = 0.02 - volume = 0.1
        # energy = 0.2 - volume = 1
        count = 0
    else:
        count = count + 1

    ind = ind + 1
    fps.update()

# stop the timer and display FPS information
fps.stop()
print("[INFO] elapsed time: {:.2f}".format(fps.elapsed()))
print("[INFO] approx. FPS: {:.2f}".format(fps.fps()))

#energy = np.mean(energy)
#print("Average energy in an execution of " + str(RECORD_SECONDS*ind) + " Seconds: " + str(energy))
stream.stop_stream()
stream.close()
streamout.stop_stream()
streamout.close()
p.terminate()
p2.terminate()


import hashlib
import time
import utils
import asyncpg
import asyncpg.exceptions as asyncpg_exc
from config import logger
from aiohttp import web
from db_wrapper import DbWrapper

router = web.RouteTableDef()

@router.post('/sign_in')
async def sign_in(request: web.Request):
    body = await request.json()
    email: str = body.get('email')
    password: str = body.get('password')
    hash = hashlib.sha256(password.encode('UTF-8')).hexdigest()
    user_data = await DbWrapper().get_user_data_by_email(email)
    if not user_data:
        return web.json_response(utils.generate_response(0, 'No account with such email'), status=403)

    if not hash == user_data['hash']:
        return web.json_response(utils.generate_response(0, 'Incorrect password'), status=403)

    # TODO send token
    response = utils.generate_response(1, 'Authorization_successful')
    token = await DbWrapper().get_token( user_data['user_id'])
    response['data'].update({'token': token['tkn'], 'timestamp': token['timestamp'], 'lifetime': token['lifetime']})
    return web.json_response(response, status=200)

@router.post('/sign_up')
async def sign_up(request: web.Request):
    body = await request.json()

    firstname: str = body.get('firstname')
    lastname: str = body.get('lastname')
    mail: str = body.get('email')
    password: str = body.get('password')
    pw_hash = hashlib.sha256(password.encode('utf-8')).hexdigest()
    try:
        user_by_email = await DbWrapper().get_user_data_by_email(mail)
        if user_by_email:
            return web.json_response(utils.generate_response(0, 'Account already exists'))

        user_insert_result = await DbWrapper().insert_user(firstname=firstname, lastname=lastname, email=mail, password_hash=pw_hash, timestamp=int(time.time()))
        # this is here because sometimes it's possible for two identical requests happening at the same time
        # causing an error due to none being inserted, as insert_user returns none because in another instance of this
        # handler being run it has already been added
        if user_insert_result:
            token = utils.generate_token()
            await DbWrapper().insert_token(user_insert_result, token, int(time.time()), 0)
            return web.json_response(utils.generate_response(1, 'Account created successfully'))
        else:
            return web.json_response(utils.generate_response(0, 'Account already exists'))

    except Exception as e:
        logger.warning(f'[Route Handlers] {e}, lineno:{e.__traceback__.tb_lineno}')
        # this will send a 500 response code
        raise

@router.get('/sessions')
async def all_sessions(request: web.Request):
    result = await DbWrapper().get_all_sessions()
    response = utils.generate_response(1, 'Session list returned')
    response['data'].update({'sessions':[]})
    for session in result:
        specializations = await DbWrapper().get_instructor_specs(session['instructor_id'])
        response['data']['sessions'].append({
            'session_id': session['session_id'],
            'session_start': session['session_start'],
            'session_place': session['session_place'],
            'session_name': session['session_name'],
            'capacity': session['capacity'],
            'signed_up': session['signed_up'],
            'firstname': session['firstname'],
            'lastname': session['lastname'],
            'specialization': [record['spec_name'] for record in specializations]
        })

    return web.json_response(response, status=200)

@router.post('/sign_up_for_session')
async def sign_for_session(request:web.Request):
    body = await request.json()
    session_id = body.get('session_id')
    uid = request.headers.get('X-User-Id')

    try:
        await DbWrapper().sign_up_for_session(user_id=uid, session_id=session_id)
    except asyncpg_exc.UniqueViolationError as e:
        return web.json_response(utils.generate_response(0, 'Double signup attempt'), status=400)
    except Exception as e:
        return web.json_response(utils.generate_response(0, f'Something went wrong. {e}'), status=500)

    return web.json_response(utils.generate_response(1, 'Success'), status=200)

@router.post('/unsign_from_session')
async def unsign_for_session(request: web.Request):

    body = await request.json()
    session_id = int(body.get('session_id'))
    uid = request.headers['X-User-Id']

    await DbWrapper().unsign_from_session(session_id = session_id, user_id=uid)

    return web.Response(status=200)

@router.get('/instructors')
async def get_instructors(request: web.Request):
    result = await DbWrapper().get_instructors()
    response = utils.generate_response(1, 'Instructor list returned')
    response['data'].update({'instructors': []})
    for instructor in result:
        response['data']['instructors'].append({
            'firstname': instructor['firstname'],
            'lastname': instructor['lastname'],
            'info': instructor['info'],
            'spec': instructor['spec_name']
        })

    return web.json_response(response, status=200)

@router.post('/sessions_by_uid')
async def get_sessions_by_uid(request: web.Request):
    body = await request.json()
    uid = int(request.headers['X-User-Id'])
    result = await DbWrapper().get_sessions_by_user(uid)
    response = utils.generate_response(1, 'Session list returned')
    response['data'].update({'sessions': []})
    for session in result:
        specializations = await DbWrapper().get_instructor_specs(session['instructor_id'])
        response['data']['sessions'].append({
            'session_id': session['session_id'],
            'session_start': session['session_start'],
            'session_place': session['session_place'],
            'session_name': session['session_name'],
            'capacity': session['capacity'],
            'signed_up': session['signed_up'],
            'firstname': session['firstname'],
            'lastname': session['lastname'],
            'specialization': [record['spec_name'] for record in specializations]
        })

    return web.json_response(response, status=200)

@router.get('/get_notifications_by_uid')
async def get_notifications_by_uid(request: web.Request):
    uid = request.headers['X-User-Id']
    notifications = await DbWrapper().get_notifications_by_user_id(uid)
    data = []
    for notification in notifications:
        new_notification = {
            'notification_id': notification['notification_id'],
            'text': notification['text']
        }
        data.append(notification_data)
    response = generate_response(1, 'Success')
    response['data'] = data
    return web.json_response(data, status=200)

@router.delete('/delete_notification_link')
async def soft_delete_notification_link(request: web.Request):
    body = await request.json()
    user_id = request.headers['user_id']
    notification_id = body['notification_id']
    try:
        await DbWrapper().unbind_notification(int(user_id), int(notification_id))
    except Exception as e:
        logger.exception('Exception during delete_notification_link:')
        response = utils.generate_response(0, 'Failed to delete notification link')
        return web.Response(response, status=500)

    return web.Response(response, status=200)

@router.get('/ping')
async def ping(request: web.Request):
    return web.Response(body='pong')


from beewin_module.database.beewinDB import Member, Result
from beewin_module.models.dataClass import MemberData
from flask_restful import Resource,  request, output_json
from common.argsParser import uid_parser, member_parser

class MemberResource(Resource):

    def get(self, uid):
        if uid == 'all':
            res: Result = Member.MemberLength()
            return res.to_dict()
        else:
            res: Result = Member.read(uid)
            return res.to_dict()

    def post(self):
        arg = member_parser.parse_args()
        _member = MemberData.from_dict(arg)
        res: Result = Member.create(_member.uid, _member)
        return res.to_dict()

    def put(self):
        arg = member_parser.parse_args()
        _member = MemberData.from_dict(arg)
        res: Result = Member.update(_member.uid, _member)
        return res.to_dict()

    def delete(self):
        arg = uid_parser.parse_args()
        res: Result = Member.delete(arg['uid'])
        return res.to_dict()

if __name__ == '__main__':
    pass


import pytest
import sacrebleu

EPSILON = 1e-4

test_sentence_level_chrf = [
    (
        'Co nás nejvíc trápí, protože lékaři si vybírají, kdo bude žít a kdo zemře.',
        ['Nejvíce smutní jsme z toho, že musíme rozhodovat o tom, kdo bude žít a kdo zemře.'],
        39.14078509,
    ),
    (
        'Nebo prostě nemají vybavení, které by jim pomohlo, uvedli lékaři.',
        ['A někdy nemáme ani potřebný materiál, abychom jim pomohli, popsali lékaři.'],
        31.22557079,
    ),
    (
        'Lapali po dechu, jejich životy skončily dřív, než skutečně začaly.',
        ['Lapali po dechu a pak jejich život skončil - dřív, než skutečně mohl začít, připomněli.'],
        57.15704367,
    ),
]

# hypothesis, reference, expected score
# >= 2.0.0: some orders are not fulfilled in epsilon smoothing (chrF++.py and NLTK)
test_cases = [
    (["abcdefg"], ["hijklmnop"], 0.0),
    (["a"], ["b"], 0.0),
    ([""], ["b"], 0.0),
    ([""], ["ref"], 0.0),
    ([""], ["reference"], 0.0),
    (["aa"], ["ab"], 8.3333),
    (["a", "b"], ["a", "c"], 8.3333),
    (["a"], ["a"], 16.6667),
    (["a b c"], ["a b c"], 50.0),
    (["a b c"], ["abc"], 50.0),
    ([" risk assessment must be made of those who are qualified and expertise in the sector - these are the scientists ."],
     ["risk assessment has to be undertaken by those who are qualified and expert in that area - that is the scientists ."], 63.361730),
    ([" Die    Beziehung zwischen  Obama und Netanjahu ist nicht gerade  freundlich. "],
     ["Das Verhältnis zwischen Obama und Netanyahu ist nicht gerade freundschaftlich."], 64.1302698),
    (["Niemand hat die Absicht, eine Mauer zu errichten"], ["Niemand hat die Absicht, eine Mauer zu errichten"], 100.0),
]

# sacreBLEU < 2.0.0 mode
# hypothesis, reference, expected score
test_cases_effective_order = [
    (["a"], ["a"], 100.0),
    ([""], ["reference"], 0.0),
    (["a b c"], ["a b c"], 100.0),
    (["a b c"], ["abc"], 100.0),
    ([""], ["c"], 0.0),
    (["a", "b"], ["a", "c"], 50.0),
    (["aa"], ["ab"], 25.0),
]

test_cases_keep_whitespace = [
    (
        ["Die Beziehung zwischen Obama und Netanjahu ist nicht gerade freundlich."],
        ["Das Verhältnis zwischen Obama und Netanyahu ist nicht gerade freundschaftlich."],
        67.3481606,
    ),
    (
        ["risk assessment must be made of those who are qualified and expertise in the sector - these are the scientists ."],
        ["risk assessment has to be undertaken by those who are qualified and expert in that area - that is the scientists ."],
        65.2414427,
    ),
]

@pytest.mark.parametrize("hypotheses, references, expected_score", test_cases)
def test_chrf(hypotheses, references, expected_score):
    score = sacrebleu.corpus_chrf(
        hypotheses, [references], char_order=6, word_order=0, beta=3,
        eps_smoothing=True).score
    assert abs(score - expected_score) < EPSILON

@pytest.mark.parametrize("hypotheses, references, expected_score", test_cases_effective_order)
def test_chrf_eff_order(hypotheses, references, expected_score):
    score = sacrebleu.corpus_chrf(
        hypotheses, [references], char_order=6, word_order=0, beta=3,
        eps_smoothing=False).score
    assert abs(score - expected_score) < EPSILON

@pytest.mark.parametrize("hypotheses, references, expected_score", test_cases_keep_whitespace)
def test_chrf_keep_whitespace(hypotheses, references, expected_score):
    score = sacrebleu.corpus_chrf(
        hypotheses, [references], char_order=6, word_order=0, beta=3,
        remove_whitespace=False).score
    assert abs(score - expected_score) < EPSILON

@pytest.mark.parametrize("hypothesis, references, expected_score", test_sentence_level_chrf)
def test_chrf_sentence_level(hypothesis, references, expected_score):
    score = sacrebleu.sentence_chrf(hypothesis, references, eps_smoothing=True).score
    assert abs(score - expected_score) < EPSILON


import numpy as np
import itertools
import multiprocessing
import threading
import subprocess
import time
import sys
import os

if len(sys.argv) != 3:
    print("input data_folder thread_num")
    exit(0)

folder = sys.argv[1]
max_spawn = int(sys.argv[2])

if not os.path.exists(folder):
    os.mkdir(folder)
os.chdir(folder)

if os.name == "nt":
    proc = "../bin/throughput.exe"
else:
    proc = "../bin/throughput.out"

task_list = []
width = [7]
n_inst = [10000]
trial = [10]
ano_prob = [0] + list(np.logspace(-6,-4, 20))
ano_life = [100, 1000]

task_list = itertools.product(width, n_inst, trial, ano_prob, ano_life)
task_list = list(task_list)
task_list = [list(map(str, val)) for val in task_list]
print(task_list)
task_list = task_list * 1000
print("max_spawn = ", max_spawn)

task_lock = threading.Lock()
task_index = 0

def spawn_and_wait(thread_id):
    global task_list
    global task_lock
    global task_index
    while True:
        finish_flag = False
        my_task_index = 0

        task_lock.acquire()
        try:
            if task_index == len(task_list):
                finish_flag = True
            else:
                my_task_index = task_index
                task_index += 1
        finally:
            task_lock.release()

        if finish_flag:
            print("thread{:2} exits".format(thread_id))
            break
        else:
            fname = "result{}.txt".format(thread_id)
            arg = [proc, fname] + task_list[my_task_index]
            print(arg)
            process = subprocess.Popen(arg)
            # process = subprocess.Popen(arg, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
            start = time.time()
            print("thread{:2} : start task {}: {}".format(thread_id, my_task_index, arg))
            process.wait()
            elapsed = time.time() - start
            print("thread{:2} : finish task {}: elp:{}".format(thread_id, my_task_index, elapsed))

thread_pool = []
for ind in range(max_spawn):
    thread = threading.Thread(target=spawn_and_wait, args=[ind])
    thread_pool.append(thread)
for thread in thread_pool:
    thread.start()
for thread in thread_pool:
    thread.join()