"""
DFS, BFS
Dijkstra for single source shortest path
SCC on Digraph
Application: 2-SAT
MST
PageRank (Markov Chains)

Connected components in undirected graph G
    Run DFS and keep track of component#

"""
from collections import defaultdict

def edges_to_adjlist(edges):
    adjlist = defaultdict(list)
    for s,t in edges:
        adjlist[s].append(t)
        adjlist[t].append(s)
    return adjlist

def classic_dfs(graph):
    visited = set()
    start = list(graph.keys())[0]
    def dfs(graph, v):
        if v not in visited:
            visited.add(v)
            for t in graph[v]:
                dfs(graph, t)
    dfs(graph, start)
    return visited

def cc_dfs0(graph):
    # input: G = (V, E) in adj list representation
    # output: vertices labelled by connected components
    cc = 0
    ccnum = {}
    visited = set()
    def explore(z): # should be named dfs
        ccnum[z] = cc
        visited.add(z)
        for t in graph[z]:
            if t not in visited:
                explore(t)
    for v in graph:
        if v not in visited:
            cc += 1
            explore(v)
    return ccnum

def cc_dfs1(graph): # base on classic_dfs
    cc = 0
    ccnum = {}
    visited = set()
    def dfs(v):
        if v not in visited:
            ccnum[v] = cc
            visited.add(v)
            for t in graph[v]:
                dfs(t)
    for v in graph:
        if v not in visited:
            cc += 1
            dfs(v)
    return ccnum

def cc_dfs2(graph): # save path, backtrack
    cc = 0
    ccnum = {}
    visited = set()
    track = {}
    def explore(z): # should be named dfs
        ccnum[z] = cc
        visited.add(z)
        for t in graph[z]:
            if t not in visited:
                track[t] = z
                explore(t)
    for v in graph:
        if v not in visited:
            cc += 1
            explore(v)
    return ccnum, track

def cc_dfs3(graph):
    # prepare for digraph
    # preorder and postorder
    clock = 1
    pre, post = {}, {}
    visited = set()
    track = {}
    def explore(z): # should be named dfs
        nonlocal clock
        pre[z] = clock
        clock += 1
        visited.add(z)
        for t in graph[z]:
            if t not in visited:
                track[t] = z
                explore(t)
        post[z] = clock
        clock += 1
    for v in graph:
        if v not in visited:
            explore(v)
    return pre, post

def test0():
    edges = [(1,2), (2,3), (1,4), (3,4), (5,6)]
    adjlist = edges_to_adjlist(edges)
    print(adjlist)
    print(classic_dfs(adjlist))
    print(cc_dfs0(adjlist))
    print(cc_dfs1(adjlist))
    print(cc_dfs2(adjlist))
    print(cc_dfs3(adjlist))

if __name__ == '__main__':
    test0()


import torch
import torch.nn as nn
from torch import optim
from multiprocessing import cpu_count
import numpy as np
import argparse
import os
import time

from networks import LightSpeech
from dataset import DataLoader, collate_fn
from dataset import LightSpeechDataset
from loss import LigthSpeechLoss
import utils
import hparams as hp

def main(args):
    # Get device
    device = torch.device('cuda'if torch.cuda.is_available()else 'cpu')

    # Define model
    model = nn.DataParallel(LightSpeech()).to(device)
    print("Model Has Been Defined")
    num_param = utils.get_param_num(model)
    print('Number of LightSpeech Parameters:', num_param)

    # Get dataset
    dataset = LightSpeechDataset()

    # Optimizer
    optimizer = torch.optim.Adam(model.parameters(),
                                 lr=hp.learning_rate,
                                 weight_decay=hp.weight_decay)

    # Criterion
    criterion = LigthSpeechLoss()

    # Load checkpoint if exists
    try:
        checkpoint = torch.load(os.path.join(
            hp.checkpoint_path, 'checkpoint_%d.pth.tar' % args.restore_step))
        model.load_state_dict(checkpoint['model'])
        optimizer.load_state_dict(checkpoint['optimizer'])
        print("\n---Model Restored at Step %d---\n" % args.restore_step)

    except:
        print("\n---Start New Training---\n")
        if not os.path.exists(hp.checkpoint_path):
            os.mkdir(hp.checkpoint_path)

    # Init logger
    if not os.path.exists(hp.logger_path):
        os.mkdir(hp.logger_path)

    # Define Some Information
    Time = np.array([])
    Start = time.clock()

    # Training
    model = model.train()

    for epoch in range(hp.epochs):
        # Get Training Loader
        training_loader = DataLoader(dataset,
                                     batch_size=hp.batch_size**2,
                                     shuffle=True,
                                     collate_fn=collate_fn,
                                     drop_last=True,
                                     num_workers=0)
        total_step = hp.epochs * len(training_loader) * hp.batch_size

        for i, batchs in enumerate(training_loader):
            for j, data_of_batch in enumerate(batchs):
                start_time = time.clock()

                current_step = i * hp.batch_size + j + args.restore_step + \
                    epoch * len(training_loader)*hp.batch_size + 1

                # Init
                optimizer.zero_grad()

                # Get Data
                character = torch.from_numpy(
                    data_of_batch["text"]).long().to(device)
                # mel_gt_target = torch.from_numpy(
                #     data_of_batch["mel_gt_target"]).float().to(device)
                mel_tac2_target = torch.from_numpy(
                    data_of_batch["mel_tac2_target"]).float().to(device)

                D = torch.from_numpy(data_of_batch["D"]).int().to(device)
                cemb = torch.from_numpy(
                    data_of_batch["cemb"]).float().to(device)

                input_lengths = torch.from_numpy(
                    data_of_batch["length_text"]).long().to(device)
                output_lengths = torch.from_numpy(
                    data_of_batch["length_mel"]).long().to(device)

                max_c_len = max(input_lengths).item()
                max_mel_len = max(output_lengths).item()

                # Forward
                mel, padd_predicted, cemb_out = model(character,
                                                      input_lengths,
                                                      output_lengths,
                                                      max_c_len,
                                                      max_mel_len, D)

                # Cal Loss
                mel_loss, similarity_loss, duration_loss = criterion(mel,
                                                                     padd_predicted, cemb_out,
                                                                     mel_tac2_target,
                                                                     D, cemb)
                total_loss = mel_loss + similarity_loss + duration_loss

                # Logger
                t_l = total_loss.item()
                m_l = mel_loss.item()
                s_l = similarity_loss.item()
                d_l = duration_loss.item()

                with open(os.path.join("logger", "total_loss.txt"), "a") as f_total_loss:
                    f_total_loss.write(str(t_l)+"\n")

                with open(os.path.join("logger", "mel_loss.txt"), "a") as f_mel_loss:
                    f_mel_loss.write(str(m_l)+"\n")

                with open(os.path.join("logger", "similarity_loss.txt"), "a") as f_s_loss:
                    f_s_loss.write(str(s_l)+"\n")

                with open(os.path.join("logger", "duration_loss.txt"), "a") as f_d_loss:
                    f_d_loss.write(str(d_l)+"\n")

                # Backward
                total_loss.backward()

                # Clipping gradients to avoid gradient explosion
                nn.utils.clip_grad_norm_(model.parameters(), 1.)

                # Update weights
                optimizer.step()

                # Print
                if current_step % hp.log_step == 0:
                    Now = time.clock()

                    str1 = "Epoch [{}/{}], Step [{}/{}]:".format(
                        epoch+1, hp.epochs, current_step, total_step)
                    str2 = "Mel Loss: {:.4f}, Duration Loss: {:.4f}, Similarity Loss: {:.4f};".format(
                        m_l, d_l, s_l)
                    str3 = "Time Used: {:.3f}s, Estimated Time Remaining: {:.3f}s.".format(
                        (Now-Start), (total_step-current_step)*np.mean(Time))

                    print("\n" + str1)
                    print(str2)
                    print(str3)

                    with open(os.path.join("logger", "logger.txt"), "a") as f_logger:
                        f_logger.write(str1 + "\n")
                        f_logger.write(str2 + "\n")
                        f_logger.write(str3 + "\n")
                        f_logger.write("\n")

                if current_step % hp.save_step == 0:
                    torch.save({'model': model.state_dict(), 'optimizer': optimizer.state_dict(
                    )}, os.path.join(hp.checkpoint_path, 'checkpoint_%d.pth.tar' % current_step))
                    print("save model at step %d ..." % current_step)

                end_time = time.clock()
                Time = np.append(Time, end_time - start_time)
                if len(Time) == hp.clear_Time:
                    temp_value = np.mean(Time)
                    Time = np.delete(
                        Time, [i for i in range(len(Time))], axis=None)
                    Time = np.append(Time, temp_value)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('--restore_step', type=int, default=0)
    args = parser.parse_args()

    main(args)


"""
Given n non-negative integers representing an elevation map where the width of each bar is 1,
compute how much water it is able to trap after raining.
Example:
    Input: [0,1,0,2,1,0,1,3,2,1,2,1]
    Output: 6
"""
from typing import List

def max_water(elevation: List[int]) -> int:
    size = len(elevation)

    left_largest = [0] * size
    right_largest = [0] * size

    for index in range(1, size):
        left_largest[index] = max(left_largest[index - 1], elevation[index - 1])

    for index in range(size - 2, -1, -1):
        right_largest[index] = max(right_largest[index + 1], elevation[index + 1])

    water = 0
    for index in range(1, size - 1):  # water at index
        water += max(
            0, min(left_largest[index], right_largest[index]) - elevation[index]
        )

    return water

if __name__ == "__main__":
    assert max_water([0, 1, 0, 2, 1, 0, 1, 3, 2, 1, 2, 1]) == 6


#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
Given a binary tree, imagine yourself standing on the right side of it, return the values of the nodes you can see ordered from top to bottom.

Example:

Input: [1,2,3,null,5,null,4]
Output: [1, 3, 4]
Explanation:

   1            <---
 /   \
2     3         <---
 \     \
  5     4       <---
"""

# Definition for a binary tree node.
class TreeNode(object):
    def __init__(self, x):
        self.val = x
        self.left = None
        self.right = None

from Queue import Queue

class Solution(object):
    def rightSideView(self, root):
        """
        :type root: TreeNode
        :rtype: List[int]
        """
        return self.ans(root)

    def ans(self, root):
        if root is None:
            return []
        queue = Queue()
        queue.put((root, 0))
        res = []
        while not queue.empty():
            node, level = queue.get()
            if level == len(res):
                res.append(None)
            res[level] = node.val

            if node.left:
                queue.put((node.left, level + 1))

            if node.right:
                queue.put((node.right, level + 1))
        return res

if __name__ == "__main__":
    from tools.construct_binary_tree import Construction

    data = [3, 9, 20, None, None, 15, 7]
    constructor = Construction(data)
    root = constructor.construct()

    print(Solution().rightSideView(root))


import random
import time
from enum import Enum
import black

def add(*nums):
    ret = 0
    for num in nums:
        ret += num
    return ret

def prod(*nums):
    ret = 1
    for num in nums:
        ret *= num
    return ret

def _equal(*nums):
    return int(len(set(nums)) <= 1)

class SpecialFunctions(str, Enum):
    TIMES = '_anonymous_times'
    PLUS = '_anonymous_plus'
    IF = 'if'
    LET = 'let'
    OR = 'or'
    AND = 'and'
    LAMBDA = 'lambda'
    IFERROR = 'iferror'

def _raise(e):
    raise e

functions = {
        'div': (lambda x, y: x / y, 'div[2:3] = 2/3'),
        'pow': (lambda x, y: x ** y, 'pow[2:3] = 8'),
        'equal': (_equal, 'equal[2:2] = 1'),
        'gt': (lambda x, y: int(x > y), 'gt[2:3] = 0'),
        'mod': (lambda x, y: x % y, 'mod[3:2] = 1'),
        'rand': (lambda : random.random(), 'rand[] = random between 0 and 1'),
        'if': (lambda x, y, z: y if x else z, 'if[1:3:4] = 4'),
        'abs': (lambda x : abs(x), 'abs[-3] = 3'),
        'time': (lambda : time.time(), 'time[] = 1680909037'),
        'error': (lambda : _raise(Exception('user requested error')), 'error[] = user requested error')
    }

lambda_example = 'let [ square_plus_two : lambda [ [ pow [ $1 : 2 ] : 2 ] ] : square_plus_two [ 3 ] ]'

special_functions = {
        SpecialFunctions.TIMES: '[2|3] = 6',
        SpecialFunctions.PLUS: '[2:3] = 5',
        SpecialFunctions.IF: 'if[1:2:3] = 2',
        SpecialFunctions.LET: 'let[apple:2:[apple:3]] = 5',
        SpecialFunctions.AND: 'and[1:0] = 0',
        SpecialFunctions.OR: 'or[1:0] = 1',
        SpecialFunctions.LAMBDA: lambda_example + ' = 11',
        SpecialFunctions.IFERROR: 'iferror[error[]:2] = 2',
    }

def evaluate_lambda(f, env, args):
    env = {**env, **{'$' + str(i + 1) : evaluate(x, env) for i, x in enumerate(args)}}
    return evaluate(f, env)

def splitIntoArgs(s):
    if '[' not in s:
        raise Exception(f'expected literal or function call but got {s if s else "empty string"}')
    arg = s.split('[')[0].lower()
    s = s.removeprefix(arg) if arg else s
    if len(s) <= 1:
        m = 'empty string' if not s else arg + s
        raise Exception(f'Failed recursing on {m}')
    if not (s[0] == '[' and s[-1] == ']'):
        raise Exception('Parentheses mismatch')
    if s == '[]':
        return [arg, []]
    args = []
    j = 1
    count_parentheses = 0
    for i in range(len(s)):
        if s[i] == '[':
            count_parentheses += 1
        elif s[i] == ']':
            count_parentheses -= 1
        elif s[i] == '|' and count_parentheses == 1:
            if arg and arg != SpecialFunctions.TIMES:
                raise(Exception('| used as argument delimiter?'))
            args.append(s[j:i])
            j = i + 1
            arg = SpecialFunctions.TIMES
        elif s[i] == ':' and count_parentheses == 1:
            if arg == SpecialFunctions.TIMES:
                raise(Exception("multiplication must be pipe delimited"))
            args.append(s[j:i])
            j = i + 1
            arg = arg if arg else SpecialFunctions.PLUS
    return [arg, args + [s[j:-1]]]

def try_literal(s, env):
    try:
        return float(s)
    except:
        pass
    try:
        return env[s]
    except:
        pass
    return None

def evaluate_outer(s):
    answer = evaluate(s.replace(' ','').replace('\n',''), {})
    if isinstance(answer, tuple):
            return 'lambda: ' + answer[0]
    else:
        return answer

def evaluate(s, env):
    literal_value = try_literal(s, env)
    if literal_value is not None:
        return literal_value
    f, args = splitIntoArgs(s)
    try:
        f = SpecialFunctions(f)
    except:
        pass
    lambdas = [k for k, v in env.items() if isinstance(v, tuple)]
    if f not in functions and f not in special_functions and f not in lambdas:
        raise(Exception(f'unknown function {f} called on {args}'))
    if f in lambdas:
        return evaluate_lambda(env[f][0], env, args)
    if f == SpecialFunctions.LAMBDA:
        if len(args) != 1:
            raise(Exception('lambda takes 1 argument'))
        return (args[0], True)
    if f == SpecialFunctions.LET:
        if len(args) != 3:
            raise(Exception('let binding takes 3 arguments'))
        if args[0].startswith('$'):
            raise(Exception('$ is reserved for lambda arguments'))
        try:
            float(args[0])
        except:
            pass
        else:
            raise(Exception("can't rename a number"))
        return evaluate(args[2], {**env, args[0]: evaluate(args[1], env)})
    if f == SpecialFunctions.TIMES:
        return prod(*[evaluate(x, env) for x in args])
    if f == SpecialFunctions.PLUS:
        return add(*[evaluate(x, env) for x in args])
    # Need special cases to not evaluate dead branches of if, and, or
    if f == SpecialFunctions.IF:
        if len(args) != 3:
            raise(Exception('if takes 3 arguments'))
        return evaluate(args[1], env) if evaluate(args[0], env) else evaluate(args[2], env)
    if f == SpecialFunctions.AND:
        if not args:
            raise(Exception("Arguments to 'and' can't be empty"))
        ret = evaluate(args[0], env)
        for arg in args[1:]:
            ret = ret and evaluate(arg, env)
        return ret
    if f == SpecialFunctions.OR:
        if not args:
            raise(Exception("Arguments to 'or' can't be empty"))
        ret = evaluate(args[0], env)
        for arg in args[1:]:
            ret = ret or evaluate(arg, env)
        return ret
    if f == SpecialFunctions.IFERROR:
        if len(args) != 2:
            raise(Exception('iferror takes 2 arguments'))
        try:
            return evaluate(args[0], env)
        except:
            return evaluate(args[1], env)
    return functions[f][0](*[evaluate(x, env) for x in args])

function_descriptions = {k: v[1] for k, v in functions.items()}

def list_functions():
    return dict(function_descriptions, **special_functions)

substitutions = {
    '[': '(',
    ']': ')',
    ':' : '+',
    'and': '_and_asdf',
    'or': '_or_asdf',
    'if': '_if_asdf',
    'lambda': '_lambda_asdf',
    '$': '_dollar_sign_asdf',
}

def format(bc):
    for k, v in substitutions.items():
        bc = bc.replace(k, v)
    bc = black.format_str(bc, mode=black.FileMode(line_length=10, magic_trailing_comma=False))
    for k, v in substitutions.items():
        bc = bc.replace(v, k)
    return bc

if __name__ == '__main__':
    print(evaluate_outer(input('')))


#!/usr/bin/python3
"""index file of REST API
"""
from api.v1.views import app_views
from flask import jsonify
from models import storage

@app_views.route('/status', strict_slashes=False)
def service_status():
    """return the status of your API"""
    return jsonify({'status': 'OK'})

@app_views.route('/stats', strict_slashes=False)
def stats():
    """retrieves the number of each objects by type"""
    cls = {
        "amenities": "Amenity",
        "cities": "City",
        "places": "Place",
        "reviews": "Review",
        "states": "State",
        "users": "User"
        }
    for k, v in cls.items():
        cls[k] = storage.count(v)
    return jsonify(cls)


# coding: utf-8

"""
    NiFi Rest API

    The Rest API provides programmatic access to command and control a NiFi instance in real time. Start and                                             stop processors, monitor queues, query provenance data, and more. Each endpoint below includes a description,                                             definitions of the expected input and output, potential response codes, and the authorizations required                                             to invoke each service.

    OpenAPI spec version: 1.19.0
    Contact: dev@nifi.apache.org
    Generated by: https://github.com/swagger-api/swagger-codegen.git
"""

from pprint import pformat
from six import iteritems
import re

class RemoteProcessGroupStatusDTO(object):
    """
    NOTE: This class is auto generated by the swagger code generator program.
    Do not edit the class manually.
    """

    """
    Attributes:
      swagger_types (dict): The key is attribute name
                            and the value is attribute type.
      attribute_map (dict): The key is attribute name
                            and the value is json key in definition.
    """
    swagger_types = {
        'group_id': 'str',
        'id': 'str',
        'name': 'str',
        'target_uri': 'str',
        'transmission_status': 'str',
        'stats_last_refreshed': 'str',
        'validation_status': 'str',
        'aggregate_snapshot': 'RemoteProcessGroupStatusSnapshotDTO',
        'node_snapshots': 'list[NodeRemoteProcessGroupStatusSnapshotDTO]'
    }

    attribute_map = {
        'group_id': 'groupId',
        'id': 'id',
        'name': 'name',
        'target_uri': 'targetUri',
        'transmission_status': 'transmissionStatus',
        'stats_last_refreshed': 'statsLastRefreshed',
        'validation_status': 'validationStatus',
        'aggregate_snapshot': 'aggregateSnapshot',
        'node_snapshots': 'nodeSnapshots'
    }

    def __init__(self, group_id=None, id=None, name=None, target_uri=None, transmission_status=None, stats_last_refreshed=None, validation_status=None, aggregate_snapshot=None, node_snapshots=None):
        """
        RemoteProcessGroupStatusDTO - a model defined in Swagger
        """

        self._group_id = None
        self._id = None
        self._name = None
        self._target_uri = None
        self._transmission_status = None
        self._stats_last_refreshed = None
        self._validation_status = None
        self._aggregate_snapshot = None
        self._node_snapshots = None

        if group_id is not None:
          self.group_id = group_id
        if id is not None:
          self.id = id
        if name is not None:
          self.name = name
        if target_uri is not None:
          self.target_uri = target_uri
        if transmission_status is not None:
          self.transmission_status = transmission_status
        if stats_last_refreshed is not None:
          self.stats_last_refreshed = stats_last_refreshed
        if validation_status is not None:
          self.validation_status = validation_status
        if aggregate_snapshot is not None:
          self.aggregate_snapshot = aggregate_snapshot
        if node_snapshots is not None:
          self.node_snapshots = node_snapshots

    @property
    def group_id(self):
        """
        Gets the group_id of this RemoteProcessGroupStatusDTO.
        The unique ID of the process group that the Processor belongs to

        :return: The group_id of this RemoteProcessGroupStatusDTO.
        :rtype: str
        """
        return self._group_id

    @group_id.setter
    def group_id(self, group_id):
        """
        Sets the group_id of this RemoteProcessGroupStatusDTO.
        The unique ID of the process group that the Processor belongs to

        :param group_id: The group_id of this RemoteProcessGroupStatusDTO.
        :type: str
        """

        self._group_id = group_id

    @property
    def id(self):
        """
        Gets the id of this RemoteProcessGroupStatusDTO.
        The unique ID of the Processor

        :return: The id of this RemoteProcessGroupStatusDTO.
        :rtype: str
        """
        return self._id

    @id.setter
    def id(self, id):
        """
        Sets the id of this RemoteProcessGroupStatusDTO.
        The unique ID of the Processor

        :param id: The id of this RemoteProcessGroupStatusDTO.
        :type: str
        """

        self._id = id

    @property
    def name(self):
        """
        Gets the name of this RemoteProcessGroupStatusDTO.
        The name of the remote process group.

        :return: The name of this RemoteProcessGroupStatusDTO.
        :rtype: str
        """
        return self._name

    @name.setter
    def name(self, name):
        """
        Sets the name of this RemoteProcessGroupStatusDTO.
        The name of the remote process group.

        :param name: The name of this RemoteProcessGroupStatusDTO.
        :type: str
        """

        self._name = name

    @property
    def target_uri(self):
        """
        Gets the target_uri of this RemoteProcessGroupStatusDTO.
        The URI of the target system.

        :return: The target_uri of this RemoteProcessGroupStatusDTO.
        :rtype: str
        """
        return self._target_uri

    @target_uri.setter
    def target_uri(self, target_uri):
        """
        Sets the target_uri of this RemoteProcessGroupStatusDTO.
        The URI of the target system.

        :param target_uri: The target_uri of this RemoteProcessGroupStatusDTO.
        :type: str
        """

        self._target_uri = target_uri

    @property
    def transmission_status(self):
        """
        Gets the transmission_status of this RemoteProcessGroupStatusDTO.
        The transmission status of the remote process group.

        :return: The transmission_status of this RemoteProcessGroupStatusDTO.
        :rtype: str
        """
        return self._transmission_status

    @transmission_status.setter
    def transmission_status(self, transmission_status):
        """
        Sets the transmission_status of this RemoteProcessGroupStatusDTO.
        The transmission status of the remote process group.

        :param transmission_status: The transmission_status of this RemoteProcessGroupStatusDTO.
        :type: str
        """

        self._transmission_status = transmission_status

    @property
    def stats_last_refreshed(self):
        """
        Gets the stats_last_refreshed of this RemoteProcessGroupStatusDTO.
        The time the status for the process group was last refreshed.

        :return: The stats_last_refreshed of this RemoteProcessGroupStatusDTO.
        :rtype: str
        """
        return self._stats_last_refreshed

    @stats_last_refreshed.setter
    def stats_last_refreshed(self, stats_last_refreshed):
        """
        Sets the stats_last_refreshed of this RemoteProcessGroupStatusDTO.
        The time the status for the process group was last refreshed.

        :param stats_last_refreshed: The stats_last_refreshed of this RemoteProcessGroupStatusDTO.
        :type: str
        """

        self._stats_last_refreshed = stats_last_refreshed

    @property
    def validation_status(self):
        """
        Gets the validation_status of this RemoteProcessGroupStatusDTO.
        Indicates whether the component is valid, invalid, or still in the process of validating (i.e., it is unknown whether or not the component is valid)

        :return: The validation_status of this RemoteProcessGroupStatusDTO.
        :rtype: str
        """
        return self._validation_status

    @validation_status.setter
    def validation_status(self, validation_status):
        """
        Sets the validation_status of this RemoteProcessGroupStatusDTO.
        Indicates whether the component is valid, invalid, or still in the process of validating (i.e., it is unknown whether or not the component is valid)

        :param validation_status: The validation_status of this RemoteProcessGroupStatusDTO.
        :type: str
        """
        allowed_values = ["VALID", "INVALID", "VALIDATING"]
        if validation_status not in allowed_values:
            raise ValueError(
                "Invalid value for `validation_status` ({0}), must be one of {1}"
                .format(validation_status, allowed_values)
            )

        self._validation_status = validation_status

    @property
    def aggregate_snapshot(self):
        """
        Gets the aggregate_snapshot of this RemoteProcessGroupStatusDTO.
        A status snapshot that represents the aggregate stats of all nodes in the cluster. If the NiFi instance is a standalone instance, rather than a cluster, this represents the stats of the single instance.

        :return: The aggregate_snapshot of this RemoteProcessGroupStatusDTO.
        :rtype: RemoteProcessGroupStatusSnapshotDTO
        """
        return self._aggregate_snapshot

    @aggregate_snapshot.setter
    def aggregate_snapshot(self, aggregate_snapshot):
        """
        Sets the aggregate_snapshot of this RemoteProcessGroupStatusDTO.
        A status snapshot that represents the aggregate stats of all nodes in the cluster. If the NiFi instance is a standalone instance, rather than a cluster, this represents the stats of the single instance.

        :param aggregate_snapshot: The aggregate_snapshot of this RemoteProcessGroupStatusDTO.
        :type: RemoteProcessGroupStatusSnapshotDTO
        """

        self._aggregate_snapshot = aggregate_snapshot

    @property
    def node_snapshots(self):
        """
        Gets the node_snapshots of this RemoteProcessGroupStatusDTO.
        A status snapshot for each node in the cluster. If the NiFi instance is a standalone instance, rather than a cluster, this may be null.

        :return: The node_snapshots of this RemoteProcessGroupStatusDTO.
        :rtype: list[NodeRemoteProcessGroupStatusSnapshotDTO]
        """
        return self._node_snapshots

    @node_snapshots.setter
    def node_snapshots(self, node_snapshots):
        """
        Sets the node_snapshots of this RemoteProcessGroupStatusDTO.
        A status snapshot for each node in the cluster. If the NiFi instance is a standalone instance, rather than a cluster, this may be null.

        :param node_snapshots: The node_snapshots of this RemoteProcessGroupStatusDTO.
        :type: list[NodeRemoteProcessGroupStatusSnapshotDTO]
        """

        self._node_snapshots = node_snapshots

    def to_dict(self):
        """
        Returns the model properties as a dict
        """
        result = {}

        for attr, _ in iteritems(self.swagger_types):
            value = getattr(self, attr)
            if isinstance(value, list):
                result[attr] = list(map(
                    lambda x: x.to_dict() if hasattr(x, "to_dict") else x,
                    value
                ))
            elif hasattr(value, "to_dict"):
                result[attr] = value.to_dict()
            elif isinstance(value, dict):
                result[attr] = dict(map(
                    lambda item: (item[0], item[1].to_dict())
                    if hasattr(item[1], "to_dict") else item,
                    value.items()
                ))
            else:
                result[attr] = value

        return result

    def to_str(self):
        """
        Returns the string representation of the model
        """
        return pformat(self.to_dict())

    def __repr__(self):
        """
        For `print` and `pprint`
        """
        return self.to_str()

    def __eq__(self, other):
        """
        Returns true if both objects are equal
        """
        if not isinstance(other, RemoteProcessGroupStatusDTO):
            return False

        return self.__dict__ == other.__dict__

    def __ne__(self, other):
        """
        Returns true if both objects are not equal
        """
        return not self == other


from __future__ import annotations
from dataclasses import dataclass
from enum import Enum
from typing import *

import numpy as np
import random

from globals import *

from sklearn import preprocessing

def average(x: list[float]) -> float:
    return sum(x) / len(x) if len(x) else 0.0

class Category(Enum):
    INCREASED = 0
    DECREASED = 1
    EXTINCT = 2

@dataclass
class InitialMassHandler:
    avg_unit_pis: float
    avg_unit_hva: float

    def get_initial_masses(self, read_ratio: str) -> tuple[float, float]:
        hva_units, pis_units = tuple([int(term)
                                      for term in read_ratio.split(':')])
        return self.avg_unit_pis * pis_units, self.avg_unit_hva * hva_units

@dataclass
class RawData:
    pis_initial: float
    hva_initial: float
    nutrient_ratio: float
    shade: int

    pis_final: float
    hva_final: float

    def transform_features(self, transform_functions: TransformFunctions) -> np.array:
        new_features: list = []

        new_features.extend(transform_functions.transform(self.pis_initial))
        new_features.extend(transform_functions.transform(self.hva_initial))
        new_features.extend(transform_functions.transform(self.nutrient_ratio))
        new_features.extend(transform_functions.transform(self.shade))

        np_new_features: np.ndarray = preprocessing.PolynomialFeatures(
            degree=transform_functions.poly_degree).fit_transform(np.array([new_features]))

        np_new_features = np_new_features.flatten()

        return np_new_features

    def categorize(self):
        self.pis_category: Category = None
        self.hva_category: Category = None

        if self.pis_final <= EXTINCTION_THRESHOLD:
            self.pis_category = Category.EXTINCT
        elif self.pis_final >= self.pis_initial:
            self.pis_category = Category.INCREASED
        else:
            self.pis_category = Category.DECREASED

        if self.hva_final <= EXTINCTION_THRESHOLD:
            self.hva_category = Category.EXTINCT
        elif self.hva_final >= self.hva_initial:
            self.hva_category = Category.INCREASED
        else:
            self.hva_category = Category.DECREASED

        if self.hva_category == Category.EXTINCT:
            self.hva_category = Category.DECREASED

@dataclass
class DataContainer:
    data_list: list[RawData]
    psi_bar: np.ndarray = None
    y: np.ndarray = None
    scaler: preprocessing.MinMaxScaler = None
    classification: str = None

    def transform_features(self, transform_functions: TransformFunctions) -> np.ndarray:
        if not self.classification:
            self.y = np.array([[point.pis_final, point.hva_final]
                               for point in self.data_list])
        else:
            self.drop_zeroes()
            self.categorize_y()
        self.psi_bar = np.array([point.transform_features(
            transform_functions) for point in self.data_list])

    def categorize_y(self):
        for point in self.data_list:
            point.categorize()
        self.y = np.array([[point.pis_category.value, point.hva_category.value]
                           for point in self.data_list])

    def get_features_and_classes(self, transform_functions: TransformFunctions, normalize: bool = False, shuffle: bool = False) -> tuple[np.ndarray, np.ndarray]:
        # if shuffle and not self.classification:
        #     random.Random(SEED).shuffle(self.data_list)
        self.transform_features(transform_functions)

        if shuffle:
            combined = np.hstack((self.psi_bar, self.y))
            random.Random(SEED).shuffle(combined)
            self.psi_bar = combined[:, :-2]
            self.y = combined[:, -2:]
            # shuffler = np.random.permutation(len(self.y))
            # self.psi_bar = self.psi_bar[shuffler]
            # self.y = self.y[shuffler]

        if normalize:
            self.scaler = preprocessing.MinMaxScaler()
            self.psi_bar = self.scaler.fit_transform(self.psi_bar)

        return self.psi_bar, self.y

    def drop_zeroes(self):
        col = self.classification
        if col == 'PIS':
            new_data_list: list[RawData] = []
            for point in self.data_list:
                if point.pis_initial != 0:
                    new_data_list.append(point)
            self.data_list = new_data_list
        elif col == 'HVA':
            new_data_list: list[RawData] = []
            for point in self.data_list:
                if point.hva_initial != 0:
                    new_data_list.append(point)
            self.data_list = new_data_list
        else:
            print('No changes made.')
            return

@dataclass
class TransformFunctions:
    poly_degree: int
    functions: list[Callable]

    def __post_init__(self):
        self.functions.append(lambda x: x)

    def transform(self, feature: float) -> list[float]:
        return [func(feature) for func in self.functions]


import torch
from torch import nn
import torch.nn.functional as F
import math
import os
from knnbox.common_utils import read_config, write_config
from knnbox.combiner.utils import calculate_combined_prob, calculate_knn_prob

class RobustCombiner(nn.Module):
    r""" Robust knn-mt Combiner """
    def __init__(self,
                max_k,
                probability_dim,
                midsize = 32,
                midsize_dc = 4,
                topk_wp = 8,
                **kwargs
                ):
        super().__init__()

        self.max_k = max_k
        self.probability_dim = probability_dim
        self.midsize = midsize
        self.midsize_dc = midsize_dc
        self.topk_wp = topk_wp
        self.kwargs = kwargs
        self.mask_for_distance = None

        self.meta_k_network = MetaKNetwork(
            max_k=self.max_k,
            midsize=self.midsize,
            midsize_dc=self.midsize_dc,
            topk_wp=self.topk_wp,
            **kwargs
        )

    def set_num_updates(self, num_updates):
        """State from trainer to pass along to model at every update."""

        def _apply(m):
            if hasattr(m, "set_num_updates") and m != self:
                m.set_num_updates(num_updates)

        self.apply(_apply)

    def get_knn_prob(
        self,
        tgt_index: torch.Tensor,
        knn_dists: torch.Tensor,
        knn_key_feature: torch.Tensor,
        network_probs: torch.Tensor,
        network_select_probs: torch.Tensor,
        device="cuda:0"
    ):
        metak_outputs = self.meta_k_network(
            tgt_index=tgt_index,
            knn_dists=knn_dists,
            knn_key_feature=knn_key_feature,
            network_probs=network_probs,
            network_select_probs=network_select_probs
        )

        self.lambda_ = metak_outputs["knn_lambda"]

        knn_prob = torch.zeros(*network_probs.shape, device=device)
        knn_prob.scatter_add_(dim=-1, index=tgt_index, src=metak_outputs["probs"])

        return knn_prob

    def get_combined_prob(self, knn_prob, neural_model_logit, log_probs=False):
        r""" get combined probs of knn_prob and neural_model_prob """
        return calculate_combined_prob(knn_prob, neural_model_logit, self.lambda_, log_probs)

    def dump(self, path):
        r""" dump the robust knn-mt to disk """
        # step 1. write config
        config = {}
        config["max_k"] = self.max_k
        config["probability_dim"] = self.probability_dim
        config["midsize"] = self.midsize
        config["midsize_dc"] = self.midsize_dc
        for k, v in self.kwargs.items():
            config[k] = v
        write_config(path, config)
        # step 2. save model
        torch.save(self.state_dict(), os.path.join(path, "robust_combiner.pt"))

    @classmethod
    def load(cls, path):
        r""" load the robust knn-mt from disk """
        config = read_config(path)
        robust_combiner = cls(**config)

        robust_combiner.load_state_dict(torch.load(os.path.join(path, "robust_combiner.pt")))
        return robust_combiner

    @staticmethod
    def _generate_mask_for_distance(max_k, device):
        k_mask = torch.empty((max_k, max_k)).fill_(999.)
        k_mask = torch.triu(k_mask, diagonal=1) + 1
        power_index = torch.tensor([pow(2, i) - 1 for i in range(0, int(math.log(max_k, 2)) + 1)])
        k_mask = k_mask[power_index]
        k_mask.requires_grad = False
        k_mask = k_mask.to(device)
        return k_mask

class MetaKNetwork(nn.Module):
    r""" meta k network of robust knn-mt """
    def __init__(
        self,
        max_k = 32,
        midsize = 32,
        midsize_dc = 4,
        topk_wp = 8,
        k_trainable = True,
        lambda_trainable = True,
        temperature_trainable = True,
        relative_label_count = False,
        device = "cuda:0",
        **kwargs,
    ):
        super().__init__()
        self.max_k = max_k
        self.k_trainable = k_trainable
        self.lambda_trainable = lambda_trainable
        self.temperature_trainable = temperature_trainable
        self.relative_label_count = relative_label_count
        self.device = device
        self.mask_for_label_count = None
        self.midsize = midsize
        self.midsize_dc = midsize_dc
        self.topk_wp = topk_wp

        # Robust kNN-MT always uses the same configuration
        # ? WP network: S_{NMT}
        self.distance_func = nn.Sequential(
            nn.Linear(self.max_k * 2 + self.topk_wp, 1), # ? W_6
        )
        # ? DC network: c_k
        self.distance_fc1 = nn.Sequential(
            nn.Linear(2, self.midsize_dc), # ? W_4
            nn.Tanh(),
            nn.Linear(self.midsize_dc, 1), # ? W_3
        )
        # ? WP network: S_{kNN}  &  WP network: T
        self.distance_fc2 = nn.Sequential(
            nn.Linear(self.max_k * 2, self.midsize), # ? W_2
            nn.Tanh(),
            nn.Linear(self.midsize, 2), # ? [W_1, W_5]
        )

    def forward(
        self,
        tgt_index: torch.Tensor,
        knn_dists: torch.Tensor,
        knn_key_feature: torch.Tensor,
        network_probs: torch.Tensor,
        network_select_probs: torch.Tensor,
    ):
        B, S, K = knn_dists.size()
        label_counts = self._get_label_count_segment(tgt_index, self.relative_label_count)
        all_key_feature = torch.cat([knn_key_feature.log().unsqueeze(-1), network_select_probs.log().unsqueeze(-1)], -1)
        top_prob, top_idx = torch.topk(network_probs, self.topk_wp)
        knn_feat = torch.cat([knn_dists, label_counts.float()], -1)
        # ? c_k
        noise_logit = self.distance_fc1(all_key_feature).squeeze(-1)
        # ? S_{NMT}
        sim_lambda = self.distance_func(torch.cat([top_prob.log(), knn_key_feature.log(), network_select_probs.log()], -1))
        # ? [S_{kNN},T]
        lambda_logit = self.distance_fc2(knn_feat.view(B, S, -1))
        # ? \lambda_t
        knn_lambda = torch.softmax(torch.cat([lambda_logit[:, :, :1], sim_lambda], -1), -1)[:, :, :1]
        # ? T
        tempe = torch.sigmoid(lambda_logit[:, :, 1:2])
        probs = torch.softmax(-knn_dists * tempe + noise_logit, -1)

        return {
            "probs": probs,
            "knn_lambda": knn_lambda,
        }

    def _get_label_count_segment(self, vals, relative=False):
        r""" this function return the label counts for different range of k nearest neighbor
            [[0:0], [0:1], [0:2], ..., ]
        """

        # caculate `label_count_mask` only once
        if self.mask_for_label_count is None:
            mask_for_label_count = torch.empty((self.max_k, self.max_k)).fill_(1)
            mask_for_label_count = torch.triu(mask_for_label_count, diagonal=1).bool()
            mask_for_label_count.requires_grad = False
            # [0,1,1]
            # [0,0,1]
            # [0,0,0]
            self.mask_for_label_count = mask_for_label_count.to(vals.device)

        ## TODO: The feature below may be unreasonable
        B, S, K = vals.size()
        expand_vals = vals.unsqueeze(-2).expand(B,S,K,K)
        expand_vals = expand_vals.masked_fill(self.mask_for_label_count, value=-1)

        labels_sorted, _ = expand_vals.sort(dim=-1) # [B, S, K, K]
        labels_sorted[:, :, :, 1:] *= ((labels_sorted[:, :, :, 1:] - labels_sorted[:, :, : , :-1]) != 0).long()
        retrieve_label_counts = labels_sorted.ne(0).sum(-1)
        retrieve_label_counts[:, :, :-1] -= 1

        if relative:
            retrieve_label_counts[:, :, 1:] = retrieve_label_counts[:, :, 1:] - retrieve_label_counts[:, :, :-1]

        return retrieve_label_counts


from __future__ import annotations

from typing import Optional

from range_typed_integers import u16

from skytemple_files.common.util import AutoString

class SsaHeader(AutoString):
    """
    The header of a read SSA file.
    Only contains information used during read-in of the data. Not used for saving.
    """

    layer_count: u16
    layer_list_pointer: Optional[int]
    trigger_pointer: Optional[int]
    actor_pointer: Optional[int]
    object_pointer: Optional[int]
    performer_pointer: Optional[int]
    events_pointer: Optional[int]
    position_marker_pointer: Optional[int]
    unk10_pointer: Optional[int]

    def __init__(
        self,
        layer_count: u16,
        layer_list_pointer: int,
        trigger_pointer: int,
        actor_pointer: int,
        object_pointer: int,
        performer_pointer: int,
        events_pointer: int,
        position_marker_pointer: int,
        unk10_pointer: int,
    ):
        # If the pointer of the following entry is the same, then this layer doesn't actually exist.
        self.layer_count = layer_count
        self.layer_list_pointer = (
            layer_list_pointer if layer_list_pointer != trigger_pointer else None
        )

        self.trigger_pointer = (
            trigger_pointer if trigger_pointer != actor_pointer else None
        )
        if self.layer_list_pointer is not None:
            self.trigger_end_pointer = actor_pointer

        self.actor_pointer = actor_pointer if actor_pointer != object_pointer else None
        if self.actor_pointer is not None:
            self.actor_end_pointer = object_pointer

        self.object_pointer = (
            object_pointer if object_pointer != performer_pointer else None
        )
        if self.object_pointer is not None:
            self.object_end_pointer = performer_pointer

        self.performer_pointer = (
            performer_pointer if performer_pointer != events_pointer else None
        )
        if self.performer_pointer is not None:
            self.performer_end_pointer = events_pointer

        self.events_pointer = (
            events_pointer if events_pointer != position_marker_pointer else None
        )
        if self.events_pointer is not None:
            self.events_end_pointer = position_marker_pointer

        self.position_marker_pointer = (
            position_marker_pointer
            if position_marker_pointer != unk10_pointer
            else None
        )
        if self.position_marker_pointer is not None:
            self.position_marker_end_pointer = unk10_pointer

        self.unk10_pointer = (
            unk10_pointer if unk10_pointer != layer_list_pointer else None
        )
        if self.unk10_pointer is not None:
            self.unk10_end_pointer = layer_list_pointer


import json
# import difflib
from difflib import get_close_matches, SequenceMatcher

data = json.load(open("projects/wordbook/data.json"))

# input                         output
# if word                       data, status=success
# elif word                     date=null, status=similar_word, meta_data=[]
    # word, yes_or_no = yes     data=[], status=success
    # word, yes_or_no = no      data=null, status=error, message
# else word                     data=null, status=error, message

def translate(word_lower):
    if word_lower == None:
        return None, "no_data", None
    word_lower = word_lower.lower()
    similar_words = get_close_matches(word_lower, data.keys())
    if word_lower in data:
        return data[word_lower], "success", None
    elif len(similar_words)>0:
        return None, "similar_word", similar_words

    else:
        return None, "unsuccessful", "Please double check the word"

# match_ratio = SequenceMatcher(None, "sagar", "sagrrr").ratio()
# print(match_ratio)

# word = input("Enter a word: ")
# output =translate()
# if type(output) == list:
#     for i in output:
#         print(i)
# else:
#     print("enter valid word")


from django.http import HttpResponse
from django.shortcuts import render_to_response, get_object_or_404
from felicasite.simpleserver.models import Card

def card(request, card_id):
    if request.method == 'GET':
        card = get_object_or_404(Card, card_id=card_id)
        return render_to_response('card.html',
                                  {'card_id': card.card_id})
    elif request.method == 'PUT':
        length_str = request.environ.get('CONTENT_LENGTH', 0)
        if length_str:
            length = int(length_str)
        else:
            length = 0

        stream = request.environ['wsgi.input']
        data = stream.read(length)

        card = Card.load_json(data)
        card.save()
        return HttpResponse(card_id)


from __future__ import print_function
import paddle
from PIL import Image
import paddle.fluid as fluid
import numpy as np
import sys
import os
import shutil
import math
import matplotlib.pyplot as plt
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import confusion_matrix
from sklearn.metrics import roc_curve, auc

def custom_image_reader(file_list, data_dir):
    q, p=file_list.shape
    image_count = q
    label_set = set()
    for i in range(q):
        img_path, label = file_list[i]
        img_path = os.path.join(data_dir, img_path)
        label_set.add(label)
        class_dim = len(label_set)
    print("class dim:{0} image count:{1}".format(class_dim, image_count))

    def reader():
        for i in range(q):
            img_path, label = file_list[i]
            img_path = os.path.join(data_dir, img_path)
            img = Image.open(img_path).convert('L')
            img = img.resize((64, 64), Image.ANTIALIAS)
            img = np.array(img).astype('float32')
            img = abs(img/255-1)
            yield img, int(label)

    return reader

def CNN(image):
    conv1_1 = fluid.layers.conv2d(input=image, num_filters=64, filter_size=3, act='relu', stride=1)
    conv1 = fluid.layers.batch_norm(input=conv1_1, act='relu')
    print(conv1.shape)
    pool1 = fluid.layers.pool2d(input=conv1, pool_size=2,pool_type='max',pool_stride=2)
    print(pool1.shape)

    conv2_1 = fluid.layers.conv2d(input=pool1, num_filters=32, filter_size=3, act='relu', stride=1)
    conv2 = fluid.layers.batch_norm(input=conv2_1, act='relu')
    print(conv2.shape)
    pool2 = fluid.layers.pool2d(input=conv2, pool_size=2,pool_type='max',pool_stride=2)
    print(pool2.shape)

    h1 = fluid.layers.fc(input=pool2, size=100, act='relu')
    #h1 =  fluid.layers.batch_norm(input=h1, act='relu')
    h1 = fluid.layers.dropout(x=h1, dropout_prob=0.5, seed=None)
    h2 = fluid.layers.fc(input=h1, size=50, act='relu')
    #h2 =  fluid.layers.batch_norm(input=h2, act='relu')
    h2 = fluid.layers.dropout(x=h2, dropout_prob=0.5, seed=None)
    prediction = fluid.layers.fc(input=h2, size=2, act='softmax')
    return prediction

final_test_acc = []
final_test_Sn = []
final_test_Sp =[]
final_mcc = []
final_auc = []
tprs = []
fprs = []
data_dir1 = "work/FCGR-4-mers"

humandata = np.loadtxt("work/FCGR-4-mers--list.txt", dtype=np.str, delimiter=' ')
np.random.shuffle(humandata)
n_splits=10
sKF = StratifiedKFold(n_splits=n_splits, shuffle=False)
i = 0

stop_train = False
num_epochs = 20
doc = open('work/print.txt','w')
for train_index, test_index in sKF.split(humandata[:,0],humandata[:,1]):
    i +=1
    train_program = fluid.Program()
    startup_prog = fluid.Program()
    with fluid.program_guard(train_program, startup_prog):
        image = fluid.layers.data(name='image', shape=[1, 64, 64], dtype='float64')
        label = fluid.layers.data(name='label', shape=[1], dtype='int64')
        with fluid.unique_name.guard():
            predict = CNN(image=image)
            cost = fluid.layers.cross_entropy(input=predict, label=label)
            avg_cost = fluid.layers.mean(cost)
            acc = fluid.layers.accuracy(input=predict, label=label)
            test_program = train_program.clone(for_test=True)
            optimizer = fluid.optimizer.AdamaxOptimizer(learning_rate=0.001,
                                                       regularization=fluid.regularizer.L2Decay(regularization_coeff=0.005))
            #optimizer = fluid.optimizer.MomentumOptimizer(learning_rate=0.001, momentum=0.98)
            opts = optimizer.minimize(avg_cost)

    use_gpu=False
    place = fluid.CUDAPlace(0) if use_gpu else fluid.CPUPlace()
    #place = fluid.CPUPlace()
    exe = fluid.Executor(place)
    # 进行参数初始化
    exe.run(startup_prog)
    feeder = fluid.DataFeeder(place=place, feed_list=[image, label])
    model_save_dir = str("/home/aistudio/model/dna%s.inference.model" %i)
    train_d = humandata[train_index]
    test_dna = humandata[test_index]
    train_dna, val_dna =train_test_split(train_d,test_size=0.1)
    train_reader = paddle.batch(custom_image_reader(train_dna, data_dir1), batch_size=64)
    val_reader = paddle.batch(custom_image_reader(val_dna, data_dir1), batch_size=64)
    test_reader = paddle.batch(custom_image_reader(test_dna, data_dir1), batch_size=64)
    test_label = list(test_dna[:,1])
    test_label = list(map(int, test_label))
    predict_label = []
    score_test = []
    stop_train = False
    Valacc=[]
    Valloss=[]
    Trainacc=[]
    Trainloss=[]
    #successive_count = 0
    for pass_id in range(num_epochs):
        train_reader=paddle.reader.shuffle(train_reader, 3000)
        for step_id, data in enumerate(train_reader()):
            train_acc, loss = exe.run(program=train_program,
                                        feed=feeder.feed(data),
                                        fetch_list=[acc,avg_cost])
            Trainacc.append(train_acc[0])
            Trainloss.append(loss[0])
        Tacc=(sum(Trainacc) / len(Trainacc))
        Tloss=(sum(Trainloss) / len(Trainloss))
        print("Pass:{0}, train_acc:{1}, train_loss:{2}" .format(pass_id, Tacc, Tloss))

    fluid.io.save_inference_model(dirname=model_save_dir,
                                  feeded_var_names=['image'],
                                  target_vars=[predict],
                                  executor=exe,
                                  main_program=train_program)

    for step, data in enumerate(test_reader()):
        out = exe.run(program=test_program,
                            feed=feeder.feed(data),
                            fetch_list=[predict])
        for item in out:
            for it in item:
                score_test.append(it[1])
                predict_label.append(np.argmax(it))
    fpr, tpr, thresholds = roc_curve(test_label,score_test)
    tprs.append(tpr)
    fprs.append(fpr)
    roc_auc = auc(fpr,tpr)
    tn, fp, fn, tp = confusion_matrix(test_label, predict_label).ravel()
    test_acc = (tn + tp)/(tn + fp + fn + tp )
    test_Sn = tp/(fn+tp)
    test_Sp = tn/(fp+tn)
    mcc = (tp*tn-fp*fn)/pow(((tp+fp)*(tp+fn)*(tn+fp)*(tn+fn)),0.5)
    final_test_acc.append(test_acc)
    final_test_Sn.append(test_Sn)
    final_test_Sp.append(test_Sp)
    final_mcc.append(mcc)
    final_auc.append(roc_auc)
    print('test_Accuracy:%0.5f, test_Sn:%0.5f, test_Sp:%0.5f, mcc:%0.5f, roc_auc:%0.5f'
                                                   % (test_acc, test_Sn, test_Sp, mcc, roc_auc))
    print("confusion matrix:\n"+str (confusion_matrix(test_label, predict_label)))
    #plt.plot(fpr,tpr,lw=1,alpha=0.3,label='ROC fold %d(area=%0.2f)'% (i , roc_auc))
    print('---------------------------------------------------')

    print('test_Accuracy:'+str(format(test_acc, '.4f'))+', test_Sn:'+str(format(test_Sn, '.4f'))+', test_Sp:'+str(format(test_Sp, '.4f'))
                                +', mcc:'+str(format(mcc, '.4f'))+', roc_auc'+str(format(roc_auc, '.4f')), file=doc)
    print("confusion matrix:\n"+str (confusion_matrix(test_label, predict_label)), file=doc)
    print('---------------------------------------------------', file=doc)

Final_test_acc = (sum(final_test_acc) / len(final_test_acc))
Final_test_Sn = (sum(final_test_Sn) / len(final_test_Sn))
Final_test_Sp = (sum(final_test_Sp) / len(final_test_Sp))
Final_mcc =  (sum(final_mcc) / len(final_mcc))
Final_auc = (sum(final_auc)/len(final_auc))
print('Final_test_Accuracy:%0.5f' % (Final_test_acc))
print('Final_test_Sn:%0.5f' % (Final_test_Sn))
print('Final_test_Sp:%0.5f' % (Final_test_Sp))
print('Final_mcc:%0.5f' % (Final_mcc))
print('Final_AUC:%0.5f' % (Final_auc))

print('Final_test_Accuracy:' + str(format(Final_test_acc, '.4f')), file=doc)
print('Final_test_Sn:' + str(format(Final_test_Sn, '.4f')), file=doc)
print('Final_test_Sp:' + str(format(Final_test_Sp, '.4f')), file=doc)
print('Final_mcc:' + str(format(Final_mcc, '.4f')), file=doc)
print('Final_AUC:' + str(format(Final_auc, '.4f')), file=doc)


import pickle
import numpy as np
from sklearn.preprocessing import MinMaxScaler

scaler = MinMaxScaler()

accuracy_ranking = pickle.load(open("/home/felix/phd/ranking_exeriments/accuracy_ranking.p", "rb"))
print(accuracy_ranking)
accuracy_ranking /= np.sum(accuracy_ranking)
#accuracy_ranking = scaler.fit_transform(accuracy_ranking.reshape(-1, 1))

fairness_ranking = pickle.load(open("/home/felix/phd/ranking_exeriments/fairness_ranking.p", "rb"))
print(fairness_ranking)
fairness_ranking /= np.sum(fairness_ranking)
#fairness_ranking = scaler.fit_transform(fairness_ranking.reshape(-1, 1))

robustness_ranking = pickle.load(open("/home/felix/phd/ranking_exeriments/robustness_ranking.p", "rb"))
print(robustness_ranking)
robustness_ranking /= np.sum(robustness_ranking)
#robustness_ranking = scaler.fit_transform(robustness_ranking.reshape(-1, 1))

names = pickle.load(open("/home/felix/phd/ranking_exeriments/names.p", "rb"))

import matplotlib
import matplotlib.pyplot as plt
import numpy as np

ids = np.argsort(accuracy_ranking,axis=0).flatten()
#ids = np.argsort(fairness_ranking,axis=0).flatten()
#ids = np.argsort(robustness_ranking,axis=0).flatten()
print(ids)
labels = np.array(names)[ids]

x = np.arange(len(labels))  # the label locations
width = 0.35  # the width of the bars

fig, ax = plt.subplots()
rects1 = ax.bar(x - width/2, accuracy_ranking.flatten()[ids], width, label='accuracy')
rects2 = ax.bar(x + 0, fairness_ranking.flatten()[ids], width, label='fairness')
rects3 = ax.bar(x + width/2, robustness_ranking.flatten()[ids], width, label='robustness')

# Add some text for labels, title and custom x-axis tick labels, etc.
ax.set_ylabel('Scores')
ax.set_title('Scores by group and gender')
ax.set_xticks(x)
ax.set_xticklabels(labels)
ax.legend()

fig.tight_layout()

plt.show()


from absl import flags
from acme.agents.jax import impala
from acme.agents.jax.impala import builder as impala_builder
import helpers
from absl import app
from acme.jax import experiments
from acme.utils import lp_utils
import launchpad as lp
import optax

# Flags which modify the behavior of the launcher.
RUN_DISTRIBUTED = flags.DEFINE_bool(
    'run_distributed', True, 'Should an agent be executed in a distributed '
    'way. If False, will run single-threaded.')
ENV_NAME = flags.DEFINE_string('env_name', 'Pong', 'What environment to run.')
SEED = flags.DEFINE_integer('seed', 0, 'Random seed (experiment).')
NUM_ACTOR_STEPS = flags.DEFINE_integer(
    'num_steps', 1_000_000,
    'Number of environment steps to run the agent for.')

_BATCH_SIZE = 32
_SEQUENCE_LENGTH = _SEQUENCE_PERIOD = 20  # Avoids overlapping sequences.

def build_experiment_config():
  """Builds IMPALA experiment config which can be executed in different ways."""

  # Create an environment, grab the spec, and use it to create networks.
  env_name = ENV_NAME.value

  def env_factory(seed):
    del seed
    return helpers.make_atari_environment(
        level=env_name,
        sticky_actions=True,
        zero_discount_on_life_loss=False,
        oar_wrapper=True)

  # Construct the agent.
  num_learner_steps = NUM_ACTOR_STEPS.value // (_SEQUENCE_PERIOD * _BATCH_SIZE)
  lr_schedule = optax.linear_schedule(2e-4, 0., num_learner_steps)
  config = impala.IMPALAConfig(
      batch_size=_BATCH_SIZE,
      sequence_length=_SEQUENCE_LENGTH,
      sequence_period=_SEQUENCE_PERIOD,
      learning_rate=lr_schedule,
      entropy_cost=5e-3,
      max_abs_reward=1.,
  )

  return experiments.ExperimentConfig(
      builder=impala_builder.IMPALABuilder(config),
      environment_factory=env_factory,
      network_factory=impala.make_atari_networks,
      seed=SEED.value,
      max_num_actor_steps=NUM_ACTOR_STEPS.value)

def main(_):
  experiment_config = build_experiment_config()
  if RUN_DISTRIBUTED.value:
    program = experiments.make_distributed_experiment(
        experiment=experiment_config,
        num_actors=4 if lp_utils.is_local_run() else 256)
    lp.launch(program, xm_resources=lp_utils.make_xm_docker_resources(program))
  else:
    experiments.run_experiment(experiment_config)

if __name__ == '__main__':
  app.run(main)


from  flask import Flask, send_file, request, abort
import os

from flask import Blueprint

from models.vt_request import getRequestParamters
from models.response_manager import ResponseManager

app = Flask(__name__)
response_manager = ResponseManager(app)

image_bp = Blueprint('images', __name__)

@image_bp.errorhandler(400)
def bad_request__error(e):
    print(f"bleu print internal_server_error {e}")
    # 针对本蓝图URL空间中的500处理
    response = response_manager.json_response_with_code({'code': '400', 'message': 'Paramters error'}, 400)
    return response

@image_bp.errorhandler(500)
def internal_server_error(e):
    print(f"bleu print internal_server_error {e}")
    # 针对本蓝图URL空间中的500处理
    response = response_manager.json_response_with_code({'code': '500', 'message': 'The server encounter a error!'}, 500)
    return response

@image_bp.route('/poster/<image_name>', methods=['GET'])
def images_poster(image_name):
    img_dir = "./static/images/poster"
    if image_name is None:
        params = getRequestParamters(request)
        if 'id' not in params:
            abort(400)
        image_name = params['id'] + ".jpg"

    img_path = os.path.join(img_dir, image_name)
    if os.path.isfile(img_path):
        return send_file(img_path, mimetype='image/jpg')
    abort(500)


import tesserocr

from PIL import Image, ImageDraw

#二值化
def binImg(img,binMax):
    img = img.convert('L')
    table = []
    for i in range(256):
        if i < binMax:
            table.append(0)
        else:
            table.append(1)

    img = img.point(table, '1')
    return img

# 二值数组
t2val = {}

#获取二值化数组
def twoValue(image, G):
    for y in range(0, image.size[1]):
        for x in range(0, image.size[0]):
            g = image.getpixel((x, y))
            if g > G:
                t2val[(x, y)] = 1
            else:
                t2val[(x, y)] = 0

# 根据一个点A的RGB值，与周围的8个点的RBG值比较，设定一个值N（0 <N <8），当A的RGB值与周围8个点的RGB相等数小于N时，此点为噪点
# G: Integer 图像二值化阀值
# N: Integer 降噪率 0 <N <8
# Z: Integer 降噪次数
# 输出
#  0：降噪成功
#  1：降噪失败
def clearNoise(image, N, Z):
    for i in range(0, Z):
        t2val[(0, 0)] = 1
        t2val[(image.size[0] - 1, image.size[1] - 1)] = 1

        for x in range(1, image.size[0] - 1):
            for y in range(1, image.size[1] - 1):
                nearDots = 0
                L = t2val[(x, y)]
                if L == t2val[(x - 1, y - 1)]:
                    nearDots += 1
                if L == t2val[(x - 1, y)]:
                    nearDots += 1
                if L == t2val[(x - 1, y + 1)]:
                    nearDots += 1
                if L == t2val[(x, y - 1)]:
                    nearDots += 1
                if L == t2val[(x, y + 1)]:
                    nearDots += 1
                if L == t2val[(x + 1, y - 1)]:
                    nearDots += 1
                if L == t2val[(x + 1, y)]:
                    nearDots += 1
                if L == t2val[(x + 1, y + 1)]:
                    nearDots += 1

                if nearDots < N:
                    t2val[(x, y)] = 1

def saveImage(filename, size):
    image = Image.new("1", size)
    draw = ImageDraw.Draw(image)

    for x in range(0, size[0]):
        for y in range(0, size[1]):
            draw.point((x, y), t2val[(x, y)])

    image.save(filename)

def getYzm(imgPath):

    #原图
    img=Image.open(imgPath)

    # #二值化阈值
    binMax=160
    # #二值化
    img=binImg(img,binMax)
    path = imgPath.replace('jpg', 'png')
    img.save(path)

    #二值化后图片
    binimg=Image.open(path)
    twoValue(binimg,binMax)
    #binimg.show()

    #降噪
    clearNoise(binimg,3,2)
    path1 = path.replace("png","jpeg")
    saveImage(path1, binimg.size)

    # 降噪后图片
    sbimg=Image.open(path1)

    res=tesserocr.image_to_text(sbimg)
    return res


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import argparse
import os
import time
from datetime import datetime

import fnmatch
import tensorflow as tf
import numpy as np

import cfg
import net
import image

def train(model_dir, image_paths):
    global_step = tf.train.get_or_create_global_step()

    # Create gray images and original images placeholders
    gray_images = tf.placeholder(tf.float32, shape=(cfg.BATCH_SIZE, 144, 160, 3))
    original_images = tf.placeholder(tf.float32, shape=(cfg.BATCH_SIZE, 144, 160, 3))

    # Build a Graph that computes the logits predictions from the inference model.
    logits = net.inference(gray_images)

    # Calculate loss
    loss = net.loss(original_images, logits)

    # Optimization
    train_op = tf.train.AdamOptimizer(learning_rate=1e-5).minimize(loss, global_step=global_step)

    # Create a saver.
    saver = tf.train.Saver(tf.global_variables())

    # Build an initialization operation to run below.
    init = tf.global_variables_initializer()

    # Start running operations on the Graph
    with tf.Session() as sess:
        sess.run(init)

        # Create summary writer
        # summary_writer = tf.summary.FileWriter(checkpoint_dir + "model", sess.graph)

        model_dir = os.path.join(model_dir, 'model')

        # Find previous model and restore it
        ckpt = tf.train.get_checkpoint_state(model_dir)
        if ckpt and ckpt.model_checkpoint_path:
            print("Restoring model...")
            try:
                saver.restore(sess, ckpt.model_checkpoint_path)
                print("Model restored")
            except ValueError:
                print("Can not restore model")

        step = tf.train.global_step(sess, global_step)
        while step < cfg.MAX_STEPS:
            start_time = time.time()
            original, gray = image.read_images(image_paths, cfg.BATCH_SIZE)
            _, loss_value = sess.run([train_op, loss], feed_dict={gray_images: gray, original_images: original})
            duration = time.time() - start_time

            assert not np.isnan(loss_value), 'Model diverged with loss = NaN'

            step += 1
            if step % 1 == 0:
                examples_per_sec = cfg.BATCH_SIZE / duration
                sec_per_batch = duration

                format_str = '%s: step %d, loss = %.2f (%.1f examples/sec; %.3f sec/batch'
                print(format_str % (datetime.now(), step, loss_value, examples_per_sec, sec_per_batch))

            # if step % 10 == 0:
            #     summary_str = sess.run(summary_op)
            #     summary_writer.add_summary(summary_str, step)

            # Save the model checkpoint periodically.
            if step % 100 == 0 or step == cfg.MAX_STEPS:
                print("Saving model")
                saver.save(sess, os.path.join(model_dir, 'model.ckpt'), global_step=global_step)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Train a colorization model.')
    parser.add_argument('data_dir',
                        help='Directory of the training images, should have two sub-directories: gray/ and images/')
    parser.add_argument('model_dir', default='./', help='Specify the directory to store the model')

    args = parser.parse_args()

    pattern = "*.png"
    image_list = []
    for dirpath, dirnames, filenames in os.walk(os.path.join(args.data_dir, 'images')):
        for filename in filenames:
            if fnmatch.fnmatch(filename, pattern):
                image_list.append(os.path.join(dirpath, filename))

    print("Found %s images" % len(image_list))

    train(args.model_dir, image_list)


import sqlite3
import os
import psycopg2
from dotenv import load_dotenv
from PIL import Image

load_dotenv()

image_directory = os.getenv("IMAGE_DIRECTORY")
database_path = os.getenv("DB_LOCAL_PATH")

# List of image paths
image_paths = os.listdir(image_directory)

image_file_types = [
    '.jpg',
    '.png',
    '.svg'
]

conn = sqlite3.connect(database_path)
cursor = conn.cursor()

for image in image_paths:
    file_extension = os.path.splitext(image)[1]

# Iterate over the image paths and insert into the database
    if file_extension in image_file_types:
        image_path_housing = f'{image_directory}\{image}'
        with open(image_path_housing, "rb") as image_file:
            image_data = image_file.read()

# checking database "name" column against the image being processed.
# If result(count)=0, it means the name is not already present in the database and the record is inserted.
        image_name = image

        cursor.execute(os.getenv("SQL_STATEMENT_SELECT_IMAGES"), (image_name,))
        result = cursor.fetchone()

        if result[0] == 0:
            cursor.execute(os.getenv("SQL_STATEMENT_INSERT_IMAGES"), (image_data, image_name))

conn.commit()
conn.close()


import json
import os
import time
from multiprocessing import Process
import pickle
from config import DIC_AGENTS, DIC_ENVS
import sys

def check_all_workers_working(list_cur_p):
    for i in range(len(list_cur_p)):
        if not list_cur_p[i].is_alive():
            return i

    return -1

def downsample(path_to_log):
    path_to_pkl = os.path.join(path_to_log, "inter_0.pkl")
    with open(path_to_pkl, "rb") as f_logging_data:
        logging_data = pickle.load(f_logging_data)
    subset_data = logging_data[::10]
    os.remove(path_to_pkl)
    with open(path_to_pkl, "wb") as f_subset:
        pickle.dump(subset_data, f_subset)

def run_wrapper(dir, one_round, run_cnt, if_gui):
    model_dir = "model/" + dir
    records_dir = "records/" + dir
    model_round = one_round
    dic_path = {}
    dic_path["PATH_TO_MODEL"] = model_dir
    dic_path["PATH_TO_WORK_DIRECTORY"] = records_dir

    with open(os.path.join(records_dir, "agent.conf"), "r") as f:
        dic_agent_conf = json.load(f)
    with open(os.path.join(records_dir, "exp.conf"), "r") as f:
        dic_exp_conf = json.load(f)
    with open(os.path.join(records_dir, "traffic_env.conf"), "r") as f:
        dic_traffic_env_conf = json.load(f)

    dic_exp_conf["RUN_COUNTS"] = run_cnt
    dic_traffic_env_conf["IF_GUI"] = if_gui

    # dump dic_exp_conf
    with open(os.path.join(records_dir, "test_exp.conf"), "w") as f:
        json.dump(dic_exp_conf, f)

    if dic_exp_conf["MODEL_NAME"] in dic_exp_conf["LIST_MODEL_NEED_TO_UPDATE"]:
        dic_agent_conf["EPSILON"] = 0  # dic_agent_conf["EPSILON"]  # + 0.1*cnt_gen
        dic_agent_conf["MIN_EPSILON"] = 0

    agent_name = dic_exp_conf["MODEL_NAME"]
    # TODO with intersection id
    agent = DIC_AGENTS[agent_name](
        dic_agent_conf=dic_agent_conf,
        dic_traffic_env_conf=dic_traffic_env_conf,
        dic_path=dic_path,
        cnt_round=0,  # useless
        # intersection_id=str(i)
    )
    if 1:
        agent.load_network(model_round)

        path_to_log = os.path.join(dic_path["PATH_TO_WORK_DIRECTORY"], "test_round", model_round)
        if not os.path.exists(path_to_log):
            os.makedirs(path_to_log)
        env = DIC_ENVS[dic_traffic_env_conf["SIMULATOR_TYPE"]](path_to_log=path_to_log,
                                                               path_to_work_directory=dic_path[
                                                                   "PATH_TO_WORK_DIRECTORY"],
                                                               dic_traffic_env_conf=dic_traffic_env_conf)

        done = False
        state = env.reset()
        step_num = 0

        while not done and step_num < int(dic_exp_conf["RUN_COUNTS"] / dic_traffic_env_conf["MIN_ACTION_TIME"]):
            action_list = []
            for one_state in state:
                action = agent.choose_action(step_num, one_state)

                action_list.append(action)

            next_state, reward, done, _ = env.step(action_list)

            state = next_state
            step_num += 1
        env.bulk_log()
        env.end_sumo()
        if not __debug__:
            path_to_log = os.path.join(dic_path["PATH_TO_WORK_DIRECTORY"], "test_round",
                                       model_round)
            # print("downsample", path_to_log)
            downsample(path_to_log)
            # print("end down")

    # except:
    #    pass
    # import sys
    # sys.stderr.write("fail to test model_%"%model_round)
    # raise SystemExit(1)

    return

def run_inference(dir, model_file_list, RUN_COUNT):
    # dir = 'cityflow'
    model_dir = "model/" + dir
    records_dir = "records/" + dir
    dic_path = {}
    dic_path["PATH_TO_MODEL"] = model_dir
    dic_path["PATH_TO_WORK_DIRECTORY"] = records_dir

    with open(os.path.join(records_dir, "agent.conf"), "r") as f:
        dic_agent_conf = json.load(f)
    with open(os.path.join(records_dir, "exp.conf"), "r") as f:
        dic_exp_conf = json.load(f)
    with open(os.path.join(records_dir, "traffic_env.conf"), "r") as f:
        dic_traffic_env_conf = json.load(f)

    # dump dic_exp_conf
    with open(os.path.join(records_dir, "test_exp.conf"), "w") as f:
        json.dump(dic_exp_conf, f)

    if dic_exp_conf["MODEL_NAME"] in dic_exp_conf["LIST_MODEL_NEED_TO_UPDATE"]:
        dic_agent_conf["EPSILON"] = 0  # dic_agent_conf["EPSILON"]  # + 0.1*cnt_gen
        dic_agent_conf["MIN_EPSILON"] = 0

    path_to_log = os.path.join(dic_path["PATH_TO_WORK_DIRECTORY"], "test_round", 'log')
    if not os.path.exists(path_to_log):
        os.makedirs(path_to_log)

    dic_traffic_env_conf["SAVEREPLAY"] = True

    env = DIC_ENVS[dic_traffic_env_conf["SIMULATOR_TYPE"]](path_to_log=path_to_log,
                                                           path_to_work_directory=dic_path["PATH_TO_WORK_DIRECTORY"],
                                                           dic_traffic_env_conf=dic_traffic_env_conf)

    state = env.reset()
    agents = [None] * dic_traffic_env_conf['NUM_AGENTS']

    for i in range(dic_traffic_env_conf['NUM_AGENTS']):
        agent_name = dic_exp_conf["MODEL_NAME"]
        agent = DIC_AGENTS[agent_name](
            dic_agent_conf=dic_agent_conf,
            dic_traffic_env_conf=dic_traffic_env_conf,
            dic_path=dic_path,
            cnt_round=0,
            best_round=None,
            # intersection_id=str(i)
        )
        agent.load_network_(model_file_list[i])
        agents[i] = agent

    step_num = 0
    while step_num < RUN_COUNT:
        action_list = []
        for i in range(dic_traffic_env_conf["NUM_AGENTS"]):
            for one_state in state:
                one_state['cur_phase'][0] = str(one_state['cur_phase'][0])
                action = agents[i].choose_action(step_num, one_state)
                action_list.append(action)

        next_state, reward, done, _ = env.step(action_list, False)

        state = next_state
        step_num += 1

def main(memo=None):
    # run name
    if not memo:
        memo = "New_sumo/pipeline_template/"

    # test run_count
    run_cnt = 3600

    # add the specific rounds in the given_round_list, like [150, 160]
    # if none, test all the round
    given_round_list = [63]

    given_traffic_list = [
        # "cross.2phases_rou01_equal_650.xml",
        # "cross.2phases_rou01_equal_600.xml",
        # "cross.2phases_rou01_equal_550.xml",
        # "cross.2phases_rou01_equal_500.xml",
        # "cross.2phases_rou01_equal_450.xml",
        # "cross.2phases_rou01_equal_400.xml",
        # "cross.2phases_rou01_equal_350.xml",
        # "cross.2phases_rou01_equal_300.xml",
    ]

    if_gui = True

    multi_process = True
    n_workers = 100
    process_list = []
    for traffic in os.listdir("records/" + memo):
        print(traffic)
        if not ".xml" in traffic:
            continue

        if traffic != "cross.2phases_rou01_equal_100.xml_12_12_19_49_03":
            continue
        test_round_dir = os.path.join("records", memo, traffic, "test_round")
        if os.path.exists(test_round_dir):
            print("exist")
            # continue
        # if traffic[0:-15] not in given_traffic_list:
        #    continue

        work_dir = os.path.join(memo, traffic)

        if given_round_list:
            for one_round in given_round_list:
                _round = "round_" + str(one_round)
                if multi_process:
                    p = Process(target=run_wrapper, args=(work_dir, _round, run_cnt, if_gui))
                    process_list.append(p)
                else:
                    run_wrapper(work_dir, _round, run_cnt, if_gui)
        else:
            train_round_dir = os.path.join("records", memo, traffic, "train_round")
            for one_round in os.listdir(train_round_dir):
                if "round" not in one_round:
                    continue

                if multi_process:
                    p = Process(target=run_wrapper, args=(work_dir, one_round, run_cnt, if_gui))
                    process_list.append(p)
                else:
                    run_wrapper(work_dir, one_round, run_cnt, if_gui)

    if multi_process:
        i = 0
        list_cur_p = []
        for p in process_list:
            if len(list_cur_p) < n_workers:
                print(i)
                p.start()
                list_cur_p.append(p)
                i += 1
            if len(list_cur_p) < n_workers:
                continue

            idle = check_all_workers_working(list_cur_p)

            while idle == -1:
                time.sleep(1)
                idle = check_all_workers_working(
                    list_cur_p)
            del list_cur_p[idle]

        for p in list_cur_p:
            p.join()

if __name__ == "__main__":
    # main()
    model_file_list = ['./model/test/round_216_inter_0.h5', './model/test/round_216_inter_1.h5',
                       './model/test/round_216_inter_2.h5', './model/test/round_216_inter_3.h5',
                       './model/test/round_216_inter_4.h5', './model/test/round_216_inter_5.h5']

    dir = 'test/anon_1_6_300_0.3_synthetic.json_08_05_19_55_50'
    NUM_COUNT = 700

    run_inference(dir, model_file_list, NUM_COUNT)


#!/usr/bin/python3
import os
import sys
import subprocess
import re
import pandas as pd
from pathlib import Path

os.chdir(os.environ['HOME']) #Change directory to homespace
os.getcwd() #Make sure the current directory is your homespace
path_1 = os.environ['HOME']
os.system('rm -f -r ~/ICA2/protein_data_output') #Remove the sub-directory protein_data
os.mkdir(path_1 + "/ICA2/protein_data_output/") #Make a new directory called protein_data
os.chdir(path_1 + "/ICA2/protein_data_output/")

def get_protein_content():  #Define a function
    os.system('rm -fr ~/ICA2/protein_data_output/error.txt')
    protein_name = input ("Enter the protein name: ") #Ask the user to input the name of the protein of interest
    protein_name_file = protein_name.replace(' ', '_') #replace the name of protein
    Taxon_ID = input ("Enter the Taxon ID: ") #Ask the user to input the taxon ID
    output_file =path_1+ f'/ICA2/protein_data_output/{protein_name_file}_{Taxon_ID}.fasta'
#esearch is used to pull up the protein data  using the variables protein_name and Taxon_ID, and output the protein data into a fasta file, this is assigned to a variable called esearch_var
#If any error is found,  2>error.txt is used to redirect the error message shown on the screen to the textfile error.txt
    esearch_var = f"esearch -db protein -query '{protein_name}[PROTEIN]' 2>error.txt NOT PARTIAL | efilter -query txid'{Taxon_ID}'[ORGANISM] 2>error.txt NOT PARTIAL | efetch -format fasta 2>error.txt NOT PARTIAL > ~/ICA2/protein_data_output/{protein_name_file}_{Taxon_ID}.fasta"
    os.system(esearch_var) # os.system commamd is used to run the variable esearch_var
    return protein_name, Taxon_ID, output_file, protein_name_file

protein_name, Taxon_ID, output_file, protein_name_file = get_protein_content()  #calling out the function get_protein_content and assigning the function to three variables protein_name, Taxon_ID and output_file.
#print a statement to let the user know the name, taxon id and the location of the chosen protein of interest.
print("The protein of interest is" , protein_name, "with taxon id" , Taxon_ID, "located in" , output_file)

with open("error.txt", "r") as file:   #Create a with loop to read the entire file error.txt
    try :   #try except for error trapping is used
        if os.stat("error.txt").st_size > 0 : #This command is used to check the content of the file, if the file contains anything,it means that there is an error in the input the used have choson
            print("invalid protein, please try again using a valid protein name and Taxon ID") #If an error is found, this statement will be printed on the screen to warn the user that the the protein they have chosen is invalid.
    except :
        print("The protein is valid")

os.chdir(os.environ['HOME']) #change directory to the homespace
path = os.environ['HOME']
path1 = (path + "/ICA2/protein_data_output")
os.chdir = (path1)
os.listdir()
my_fasta_file = open(output_file) #open a connection (output_file) and assign it to a variable called my_fasta_file which refers to the file that contains the protein information
protein_id = [] #Make an empty list and assign it to a variable called  protein_id
protein_name = [] #Make an empty list and assign it to a variable called protein_name
organism = [] #Make an empty list and assign it to a variable called species
protein_sequence = [] #Make an empty list and assign it to a variable called protein_sequence

for eachline in my_fasta_file :
    if eachline.startswith(">") :
        protein_id_search = eachline.split()[0]
        print(protein_id_search)
        protein_id.append(protein_id_search)
        protein_headerline =  re.search('(^>)''(.+)''(\[.+\])', eachline)
        organism_search = re.search('(\[.+\])', eachline).group(0)
        print(organism_search)
    if organism_search :
        organism.append(organism_search)
        id_removed = eachline.replace(protein_id_search, '')
        protein_name_search = id_removed.replace(organism_search, '')
        print(protein_name_search)
    if protein_name_search :
        protein_name.append(protein_name_search)
    else :
        if protein_sequnece_search :
            protein_sequence_search = protein_sequence.append(eachline)
            print(protein_sequence_search)

#Make a dataframe
s1 = pd.Series(protein_id)
s2 = pd.Series(protein_name)
s3 = pd.Series(organism_search)
s4 = pd.Series(protein_sequence,dtype=pd.StringDtype())
df = pd.DataFrame ({'protein_id' : s1, 'protein_name' : s2, 'organism' : s3, 'protein_sequence' : s4 })
print(df)
csv_convert = path+ '/ICA2'
df.to_csv(path+'/ICA2/'+protein_name_file+'_'+Taxon_ID+'.csv', sep='\t')

organism_count = df['organism'].value_counts()
print(organism_count)

#Running Clustalo
path = os.environ['HOME'] #The homespace is stored into a variable called path

my_fasta_file_original = path + "/ICA2/protein_data_output/"+protein_name_file+"_"+Taxon_ID+".fasta" #The path variable and the string is assigned to a variable my_fasta_file_original, which contains the fasta file {protein_name_file}_{Taxon_ID}.fasta
my_output_file_clustalo = path + f"/ICA2/protein_data_output_aligned" #The path variable and the string containing the file protein_data_output_aligned is assigned to a variable called my_output_file_aligned, which contains the data generated by clustalo in msf form.
#clustalo command is put into a string and run using an input file (my_fasta_file_original) and the output is put into a file called my_output_file_clustalo in msf form, the string is then assigned to the variable clustalo_cmd
clustalo_cmd = f'clustalo -i {my_fasta_file_original} -o {my_output_file_clustalo} -v --outfmt msf --force'

def clustalo_user_input(question= 'Do you want to run Clustalo for sequence alignement?'): #Defining a function
    while "The answer to run clustalo is invalid" : #A while loop is used to prompt the user for input if they give an invalid answer
        clustalo_user_reply = str(input(question+ '[y/n]:')).lower().strip() #Ask the user for input (y/n) for running clustalo analysis
#If loop is used, giving instructions to run clustalo_cmd variable (clustalo analysis) if the user input is y
        if clustalo_user_reply[:1] == 'y':
            os.system(clustalo_cmd) #os.system command is used to run the variable clustalo_cmd
            print("Clustalo analysis is done, please see the results") #A statement is shown on the screen to update the user that the clustalo analysis is done
            return True #Exiting the function
#If loop is used, giving instructions to exit clustalo idf the user input to the question is n
        if clustalo_user_reply[:1] == 'n':
            print ("Exiting Clustalo .....") #A statement updating the user that clustalo analysis is not done
            return False #Exiting the function

clustalo_user_input(question= 'Do you want to run Clustalo for sequence alignement?') #Calling the function

#Running infoalign
os.system('rm -fr ~/ICA2/infoalign_output/')
os.mkdir(path + "/ICA2/infoalign_output/") # +protein_name_file+"_"+Taxon_ID+".infpalign")
my_output_file_infoalign = path + "/ICA2/infoalign_output/"  #+protein_name_file+"_"+Taxon_ID+".infoalign"
infoalign_cmd = f'infoalign -sequence {my_output_file_clustalo} -odirectory2 {my_output_file_infoalign} -'

def infoalign_user_input(question= 'Do you want to run infoalign to learn more information about the asligned sequences?'):
    while "The answer to run infoalign is invalid" :
        infoalign_user_reply = str(input(question+ '[y/n]:')).lower().strip()
        if infoalign_user_reply[:1] == 'y':
            os.system(infoalign_cmd)
            print("infoalign analysis is done, please view the results in the output file")
            return True
        if infoalign_user_reply[:1] == 'n':
            print("Exiting infoalign....")
            return False
infoalign_user_input(question= 'Do you want to run infoalign to learn more information about the aligned sequences?')

#Running Plotcon
my_output_file_clustalo = path + f"/ICA2/protein_data_output_aligned" # The variable (path) and the string containing the file protein_data_output_aligned are assigned to the variable my_output_file_clustalo
plotcon_cmd = f'plotcon -sformat -msf {my_output_file_clustalo} -winsize 4 -graph x11' #plotcon command is put into a string and run using the output file (msf form) generated by clustalo, and this is assigned to the variable plotcon_cmd. The window size is set to 4 and the graph type is set to x11 by default.

def plotcon_user_input(question= 'Do you want to view the plot now using plotcon?'): #Defining a function
    while "The answer to view the plot is invalid" : # A while loop is used to prompt the user for input if the answer they gave is invalid
        plotcon_reply = str(input(question+ '[y/n]:')).lower().strip() #Ask the user for input (y/n) to your question
#If loop is used, giving instructions to run the plotcon_cmd variable if the user input is y and this shows the plot of the protein selected on the screen.
        if plotcon_reply[:1] == 'y':
            os.system(plotcon_cmd)
            print("plotcon is currently running and the plot will be shown on the screen, please wait")
            return True #exiting the function
#If loop is used, giving instructions to not run (exit) plotcon if the user input is n
        if plotcon_reply[:1] == 'n':
            print("Exiting plotcon.....")
            return False #exiting the function
plotcon_user_input(question= 'Do you want to view the plot now using plotcon') #run the function plotcon_user_input

#Running prosite
#The single fasta file will be split into seperate fasta files with  individual protein sequences first by seqretsplit.
#Running seqretsplit
my_fasta_file_original = (os.environ['HOME'] + "/ICA2/protein_data_output/"+protein_name_file+"_"+Taxon_ID+".fasta") #The path variable and the string is assigned to a variable my_fasta_file_original, which contains the fasta file {protein_name_file}_{Taxon_ID}.
seqretsplit_output = (os.environ['HOME'] + "/ICA2/prosite_output/seqretsplit_results/") #variable seqretsplit_output is created to be used when running seqretsplit command line, as all the fasta files containing individual sequences will be assigned to this variable.
os.system(f'rm -f -r {seqretsplit_output}') #remove the variable seqretsplit_output
os.makedirs( seqretsplit_output ) #make a directory with the path assigned to variable seqretsplit_cmd
#Assign the  seqretsplit command to the variable seqretsplit_cmd, put the command line in a string and split the single fasta file in my_fasta_file_original variable into individual sequences (in seperate fasta files) that can be scanned by patmatmotifs. The fasta files generated will be put into seqretsplit_output
seqretsplit_cmd = f"seqretsplit -sequence {my_fasta_file_original} -osdirectory2 {seqretsplit_output} -"
os.system( seqretsplit_cmd) #Run the seqretsplit_cmd

#Scanning protein sequences using patmatmotifs
prosite_results = (os.environ['HOME'] + "/ICA2/prosite_output/patmotifs_output/") #Assign the directory that ends with .patmatmotifs into a variable called prosite_results
os.system(f'rm -f -r {prosite_results}') #remove the variable prosite_results
os.makedirs( prosite_results ) #make a directory using the variable prosite_results
prosite_files = Path(seqretsplit_output).glob('*')
for file in prosite_files:
    prosite_cmd = f"patmatmotifs -sequence {file} -rdirectory2 {prosite_results} -"
    os.system(prosite_cmd)


from datetime import datetime
from pathlib import Path
from typing import List, Tuple

import bs4 as bs
import nltk
import pandas as pd
import pytz
import requests
import traceback
import toml
from tqdm import tqdm

from get_data.financial import fetch_and_save_financials
from get_data.ohlcv import fetch_and_save_klines
from get_data.sentiment import fetch_and_save_sentiment

def sync_symbols(path_to_stock_symbols: Path):
    """Sync symbols with active symbols of S&P500, while keeping the stocks manually set

    Args:
        path_to_stock_symbols (Path): Path to the DataFrame of symbols
    """
    original_symbols = pd.read_csv(path_to_stock_symbols)
    force_watch_symbols = original_symbols[original_symbols["force_watch"]]

    resp = requests.get("http://en.wikipedia.org/wiki/List_of_S%26P_500_companies")
    soup = bs.BeautifulSoup(resp.text, "lxml")
    table = soup.find("table", {"class": "wikitable sortable"})

    current_symbols = []
    for row in table.findAll("tr")[1:]:
        ticker = row.findAll("td")[0].text
        current_symbols.append(ticker.replace("\n", ""))
    current_symbols = sorted(current_symbols)

    no_force_watch_symbols = pd.DataFrame(current_symbols, columns=["symbol"])
    no_force_watch_symbols["force_watch"] = False
    no_force_watch_symbols["from_date"] = datetime.today().strftime("%Y-%m-%d")

    new_symbols = pd.concat([force_watch_symbols, no_force_watch_symbols], axis=0).sort_values(
        by="symbol", axis=0
    )
    new_symbols.loc[new_symbols["symbol"].isin(original_symbols["symbol"]), "from_date"] = original_symbols["from_date"]
    new_symbols.to_csv(path_to_stock_symbols, index=False)

def update_data(
    index_symbols: pd.DataFrame,
    stock_symbols: pd.DataFrame,
    path_to_datasets: Path,
) -> Tuple[List[str], List[str], List[str]]:
    """Update the `path_to_datasets` folder by fetching the financials, sentiment score and klines of the assets

    Args:
        index_symbols (pd.DataFrame): list of indices to update
        stock_symbols (pd.DataFrame): list of stocks to update
        path_to_datasets (Path): path of the datasets to update

    Returns:
        Tuple[List[str], List[str], List[str]]: tuple made of
            * list of symbols having problems when fetching their klines
            * list of symbols having problems when fetching their sentiments
            * list of symbols having problems when fetching their financials
    """
    nltk.downloader.download("vader_lexicon")

    problematic_ohlcv = []
    problematic_sentiment = []
    problematic_financials = []

    pbar = tqdm(total=len(index_symbols) + 3 * len(stock_symbols))

    for _, row in stock_symbols.iterrows():
        symbol = row["symbol"]
        from_date = row["from_date"]
        try:
            financials = fetch_and_save_financials(
                symbol=symbol,
                directory=path_to_datasets / "financial",
            )
        except Exception as e:
            print(f"Problem fetching {symbol} financials")
            print(traceback.format_exc())
            problematic_financials.append(symbol)
        pbar.update(1)
        # try:
        #     sentiments = fetch_and_save_sentiment(
        #         symbol=symbol,
        #         beginning_date=datetime(2021, 1, 1),
        #         interval="1d",
        #         directory=path_to_datasets / "sentiment",
        #     )
        # except Exception as e:
        #     print(f"Problem fetching {symbol} sentiment")
        #     print(traceback.format_exc())
        #     problematic_sentiment.append(symbol)
        pbar.update(1)
        # try:
        #     klines = fetch_and_save_klines(
        #         symbol=symbol,
        #         beginning_date=from_date,
        #         interval="1d",
        #         directory=path_to_datasets / "ohlcv",
        #     )
        # except Exception as e:
        #     print(f"Problem fetching {symbol} klines")
        #     print(traceback.format_exc())
        #     problematic_ohlcv.append(symbol)
        pbar.update(1)
    for _, row in index_symbols.iterrows():
        symbol = row["symbol"]
        from_date = row["from_date"]
        try:
            klines = fetch_and_save_klines(
                symbol=symbol,
                beginning_date=datetime(2021, 1, 1),
                interval="1d",
                directory=path_to_datasets / "ohlcv",
            )
        except Exception as e:
            print(f"Problem fetching {symbol} klines")
            print(traceback.format_exc())
            problematic_ohlcv.append(symbol)
        pbar.update(1)

    pbar.close()
    return problematic_ohlcv, problematic_sentiment, problematic_financials

if __name__ == "__main__":
    current_datetime_utc = datetime.utcnow().replace(tzinfo=pytz.utc)
    current_datetime_nyc = current_datetime_utc.astimezone(pytz.timezone("US/Eastern"))

    print("Current UTC time:", current_datetime_utc)
    print("Current NYC / Market time:", current_datetime_nyc)
    config = toml.load(Path("config.toml"))

    path_to_index_symbols = Path(config["data_access"]["path_to_index_symbols"])
    path_to_stock_symbols = Path(config["data_access"]["path_to_stock_symbols"])
    path_to_datasets = Path(config["data_access"]["path_to_datasets"])

    # sync active symbols
    sync_symbols(path_to_stock_symbols)

    index_symbols = pd.read_csv(path_to_index_symbols)
    index_symbols["from_date"] = pd.to_datetime(index_symbols["from_date"])
    stock_symbols = pd.read_csv(path_to_stock_symbols)
    stock_symbols["from_date"] = pd.to_datetime(stock_symbols["from_date"])
    # update assets
    problematic_ohlcv, problematic_sentiment, problematic_financials = update_data(
        index_symbols,
        stock_symbols,
        path_to_datasets,
    )