import asyncio
import time
import unittest
from collections import deque
from decimal import Decimal
from typing import Union

from hummingbot.client.config.client_config_map import ClientConfigMap
from hummingbot.client.config.config_helpers import ClientConfigAdapter
from hummingbot.connector.test_support.mock_paper_exchange import MockPaperExchange
from hummingbot.core.data_type.common import OrderType, TradeType
from hummingbot.core.data_type.limit_order import LimitOrder
from hummingbot.core.data_type.market_order import MarketOrder
from hummingbot.core.event.events import (
    BuyOrderCompletedEvent,
    BuyOrderCreatedEvent,
    FundingPaymentCompletedEvent,
    MarketEvent,
    MarketOrderFailureEvent,
    OrderCancelledEvent,
    OrderExpiredEvent,
    OrderFilledEvent,
    SellOrderCompletedEvent,
    SellOrderCreatedEvent,
)
from hummingbot.strategy.market_trading_pair_tuple import MarketTradingPairTuple
from hummingbot.strategy.strategy_py_base import StrategyPyBase

class MockPyStrategy(StrategyPyBase):

    def __init__(self):
        super().__init__()

        # Used the check the events are recorded
        self.events_queue = deque()

    def did_create_buy_order(self, order_created_event: BuyOrderCreatedEvent):
        self.events_queue.append(order_created_event)

    def did_create_sell_order(self, order_created_event: SellOrderCreatedEvent):
        self.events_queue.append(order_created_event)

    def did_fill_order(self, order_filled_event: OrderFilledEvent):
        self.events_queue.append(order_filled_event)

    def did_fail_order(self, order_failed_event: MarketOrderFailureEvent):
        self.events_queue.append(order_failed_event)

    def did_cancel_order(self, cancelled_event: OrderCancelledEvent):
        self.events_queue.append(cancelled_event)

    def did_expire_order(self, expired_event: OrderExpiredEvent):
        self.events_queue.append(expired_event)

    def did_complete_buy_order(self, order_completed_event: BuyOrderCompletedEvent):
        self.events_queue.append(order_completed_event)

    def did_complete_sell_order(self, order_completed_event: SellOrderCompletedEvent):
        self.events_queue.append(order_completed_event)

    def did_complete_funding_payment(self, funding_payment_completed_event: FundingPaymentCompletedEvent):
        self.events_queue.append(funding_payment_completed_event)

class StrategyPyBaseUnitTests(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        cls.ev_loop = asyncio.get_event_loop()
        cls.trading_pair = "COINALPHA-HBOT"

    def setUp(self):
        self.market: MockPaperExchange = MockPaperExchange(
            client_config_map=ClientConfigAdapter(ClientConfigMap())
        )
        self.market_info: MarketTradingPairTuple = MarketTradingPairTuple(
            self.market, self.trading_pair, *self.trading_pair.split("-")
        )

        self.strategy: StrategyPyBase = MockPyStrategy()
        self.strategy.add_markets([self.market])

    @staticmethod
    def simulate_order_created(market_info: MarketTradingPairTuple, order: Union[LimitOrder, MarketOrder]):
        event_tag = MarketEvent.BuyOrderCreated if order.is_buy else MarketEvent.SellOrderCreated
        event_class = BuyOrderCreatedEvent if order.is_buy else SellOrderCreatedEvent

        market_info.market.trigger_event(
            event_tag,
            event_class(
                int(time.time() * 1e3),
                OrderType.LIMIT if isinstance(order, LimitOrder) else OrderType.MARKET,
                order.trading_pair,
                order.quantity if isinstance(order, LimitOrder) else order.amount,
                order.price,
                order.client_order_id if isinstance(order, LimitOrder) else order.order_id,
                time.time()
            )
        )

    @staticmethod
    def simulate_order_filled(market_info: MarketTradingPairTuple, order: Union[LimitOrder, MarketOrder]):
        market_info.market.trigger_event(
            MarketEvent.OrderFilled,
            OrderFilledEvent(
                time.time(),
                order.client_order_id if isinstance(order, LimitOrder) else order.order_id,
                order.trading_pair,
                TradeType.BUY if order.is_buy else TradeType.SELL,
                OrderType.LIMIT if isinstance(order, LimitOrder) else OrderType.MARKET,
                order.price,
                order.quantity if isinstance(order, LimitOrder) else order.amount,
                Decimal("1")
            )
        )

    @staticmethod
    def simulate_order_failed(market_info: MarketTradingPairTuple, order: Union[LimitOrder, MarketOrder]):
        market_info.market.trigger_event(
            MarketEvent.OrderFailure,
            MarketOrderFailureEvent(
                int(time.time() * 1e3),
                order.client_order_id if isinstance(order, LimitOrder) else order.order_id,
                OrderType.LIMIT if isinstance(order, LimitOrder) else OrderType.MARKET
            )
        )

    @staticmethod
    def simulate_cancel_order(market_info: MarketTradingPairTuple, order: Union[LimitOrder, MarketOrder]):
        market_info.market.trigger_event(
            MarketEvent.OrderCancelled,
            OrderCancelledEvent(
                int(time.time() * 1e3),
                order.client_order_id if isinstance(order, LimitOrder) else order.order_id,
            )
        )

    @staticmethod
    def simulate_order_expired(market_info: MarketTradingPairTuple, order: Union[LimitOrder, MarketOrder]):
        market_info.market.trigger_event(
            MarketEvent.OrderExpired,
            OrderExpiredEvent(
                int(time.time() * 1e3),
                order.client_order_id if isinstance(order, LimitOrder) else order.order_id,
            )
        )

    @staticmethod
    def simulate_order_completed(market_info: MarketTradingPairTuple, order: Union[LimitOrder, MarketOrder]):
        event_tag = MarketEvent.BuyOrderCompleted if order.is_buy else MarketEvent.SellOrderCompleted
        event_class = BuyOrderCompletedEvent if order.is_buy else SellOrderCompletedEvent

        market_info.market.trigger_event(
            event_tag,
            event_class(
                int(time.time() * 1e3),
                order.client_order_id if isinstance(order, LimitOrder) else order.order_id,
                order.trading_pair.split("-")[0],
                order.trading_pair.split("-")[1],
                Decimal("1") if order.is_buy else Decimal("0"),
                Decimal("0") if order.is_buy else Decimal("1"),
                OrderType.LIMIT if isinstance(order, LimitOrder) else OrderType.MARKET,
            )
        )

    @staticmethod
    def simulate_funding_payment_completed(market_info: MarketTradingPairTuple):

        example_rate: Decimal = Decimal("100")

        # Example API response for funding payment details
        response = {
            "symbol": "BTCUSDT",
            "incomeType": "COMMISSION",
            "income": "-0.01000000",
            "asset": "USDT",
            "info": "COMMISSION",
            "time": 1570636800000,
            "tranId": "9689322392",
            "tradeId": "2059192"
        }

        market_info.market.trigger_event(
            MarketEvent.FundingPaymentCompleted,
            FundingPaymentCompletedEvent(
                response["time"],
                market_info.market.name,
                example_rate,
                response["symbol"],
                response["income"]
            )
        )

    def test_did_create_buy_order(self):
        limit_order: LimitOrder = LimitOrder(client_order_id="test",
                                             trading_pair=self.trading_pair,
                                             is_buy=True,
                                             base_currency=self.trading_pair.split("-")[0],
                                             quote_currency=self.trading_pair.split("-")[1],
                                             price=Decimal("100"),
                                             quantity=Decimal("50"))
        self.simulate_order_created(self.market_info, limit_order)

        event = self.strategy.events_queue.popleft()

        self.assertIsInstance(event, BuyOrderCreatedEvent)

    def test_did_create_sell_order(self):
        limit_order: LimitOrder = LimitOrder(client_order_id="test",
                                             trading_pair=self.trading_pair,
                                             is_buy=False,
                                             base_currency=self.trading_pair.split("-")[0],
                                             quote_currency=self.trading_pair.split("-")[1],
                                             price=Decimal("100"),
                                             quantity=Decimal("50"))

        self.simulate_order_created(self.market_info, limit_order)

        event = self.strategy.events_queue.popleft()

        self.assertIsInstance(event, SellOrderCreatedEvent)

    def test_did_fill_order(self):
        limit_order: LimitOrder = LimitOrder(client_order_id="test",
                                             trading_pair=self.trading_pair,
                                             is_buy=False,
                                             base_currency=self.trading_pair.split("-")[0],
                                             quote_currency=self.trading_pair.split("-")[1],
                                             price=Decimal("100"),
                                             quantity=Decimal("50"))

        self.simulate_order_filled(self.market_info, limit_order)

        event = self.strategy.events_queue.popleft()

        self.assertIsInstance(event, OrderFilledEvent)

    def test_did_cancel_order(self):
        limit_order: LimitOrder = LimitOrder(client_order_id="test",
                                             trading_pair=self.trading_pair,
                                             is_buy=True,
                                             base_currency=self.trading_pair.split("-")[0],
                                             quote_currency=self.trading_pair.split("-")[1],
                                             price=Decimal("100"),
                                             quantity=Decimal("50"))

        self.simulate_cancel_order(self.market_info, limit_order)

        event = self.strategy.events_queue.popleft()

        self.assertIsInstance(event, OrderCancelledEvent)

    def test_did_fail_order(self):
        limit_order: LimitOrder = LimitOrder(client_order_id="test",
                                             trading_pair=self.trading_pair,
                                             is_buy=False,
                                             base_currency=self.trading_pair.split("-")[0],
                                             quote_currency=self.trading_pair.split("-")[1],
                                             price=Decimal("100"),
                                             quantity=Decimal("50"))

        self.simulate_order_failed(self.market_info, limit_order)

        event = self.strategy.events_queue.popleft()

        self.assertIsInstance(event, MarketOrderFailureEvent)

    def test_did_expire_order(self):
        limit_order: LimitOrder = LimitOrder(client_order_id="test",
                                             trading_pair=self.trading_pair,
                                             is_buy=False,
                                             base_currency=self.trading_pair.split("-")[0],
                                             quote_currency=self.trading_pair.split("-")[1],
                                             price=Decimal("100"),
                                             quantity=Decimal("50"))

        self.simulate_order_expired(self.market_info, limit_order)

        event = self.strategy.events_queue.popleft()

        self.assertIsInstance(event, OrderExpiredEvent)

    def test_did_complete_buy_order(self):
        limit_order: LimitOrder = LimitOrder(client_order_id="test",
                                             trading_pair=self.trading_pair,
                                             is_buy=True,
                                             base_currency=self.trading_pair.split("-")[0],
                                             quote_currency=self.trading_pair.split("-")[1],
                                             price=Decimal("100"),
                                             quantity=Decimal("50"))

        self.simulate_order_completed(self.market_info, limit_order)

        event = self.strategy.events_queue.popleft()

        self.assertIsInstance(event, BuyOrderCompletedEvent)

    def test_did_complete_sell_order(self):
        limit_order: LimitOrder = LimitOrder(client_order_id="test",
                                             trading_pair=self.trading_pair,
                                             is_buy=False,
                                             base_currency=self.trading_pair.split("-")[0],
                                             quote_currency=self.trading_pair.split("-")[1],
                                             price=Decimal("100"),
                                             quantity=Decimal("50"))

        self.simulate_order_completed(self.market_info, limit_order)

        event = self.strategy.events_queue.popleft()

        self.assertIsInstance(event, SellOrderCompletedEvent)

    def test_did_complete_funding_payment(self):
        self.simulate_funding_payment_completed(self.market_info)

        event = self.strategy.events_queue.popleft()

        self.assertIsInstance(event, FundingPaymentCompletedEvent)


#mcandrew

import sys
import numpy as np
import pandas as pd
import datetime

def computeMostRecentSurvWeek():
        return pd.read_csv('./data/epiData.csv').EW.max()

def cap2MostRecentSurvWeek(d,MRW):
    return d.loc[d.EW==MRW,:]

def formatBins(d):
    reformatted = []
    for x in d.loc[:,'bin_start_incl']:
        try:
            reformatted.append(float(x))
        except:
            reformatted.append(x)
    d.loc[:,'bin_start_incl'] = [str(x) for x in reformatted]

    reformatted = []
    for x in d.loc[:,'bin_end_notincl']:
        try:
            reformatted.append(float(x))
        except:
            reformatted.append(x)
    d.loc[:,'bin_end_notincl'] = [str(x) for x in reformatted]
    return d

def formatEnsembleForecast(d):

    d = d.rename(columns = {'location':'Location'
                            ,'target':'Target'
                            ,'type':'Type'
                            ,'unit':'Unit'
                            ,'bin_start_incl':'Bin_start_incl'
                            ,'bin_end_notincl':'Bin_end_notincl' })
    d['Value'] = d['ensembleForecast']

    d.loc[d.Location=='nat','Location'] = 'US National'
    for n in np.arange(1,10+1):
        d.loc[d.Location=='hhs{:d}'.format(n),'Location'] = 'HHS Region {:d}'.format(n)

    return d.loc[:,['Location','Target','Type','Unit','Bin_start_incl','Bin_end_notincl','Value']  ]

def sortEnsembleForecast(d):

    leftBin = []
    for x in d.Bin_start_incl:
        if x=='none':
            leftBin.append(-1.)
        else:
            leftBin.append(float(x))
    d['Bin_start_incl'] = leftBin

    rightBin = []
    for x in d.Bin_end_notincl:
        if x=='none':
            rightBin.append(-2.)
        else:
            rightBin.append(float(x))
    d['Bin_end_notincl'] = rightBin

    d['Bin_start_incl']  = d.Bin_start_incl.astype(float)
    d['Bin_end_notincl'] = d.Bin_end_notincl.astype(float)
    d = d.sort_values(['Location','Target','Type','Bin_start_incl','Bin_end_notincl'])

    d['Bin_start_incl'] = d.Bin_start_incl.astype(str)
    d['Bin_end_notincl'] = d.Bin_end_notincl.astype(str)

    d.loc[d.Bin_start_incl =='-1','Bin_start_incl']  = 'none'
    d.loc[d.Bin_end_notincl=='-2','Bin_end_notincl'] = 'none'

    return d

def captureSubmissionInformation(mostRecentSurviellanceWeek):
    EW = int(str(mostRecentSurviellanceWeek)[4:])
    today = datetime.datetime.today()
    year,month,day  = today.year,today.month,today.day
    return EW,year,month,day

def renormalizeWeightsForModelsThatDontSubmit(d):
        subsetModelsWeights = d.loc[:,['model','weight']].drop_duplicates()

        print("Sum of weights before renorm = {:f}".format(subsetModelsWeights.weight.sum()))

        subsetModelsWeights = subsetModelsWeights.rename(columns={'weight':'renormWeight'})

        sumWeights = subsetModelsWeights.renormWeight.sum()
        subsetModelsWeights['renormWeight'] = subsetModelsWeights.renormWeight/sumWeights

        d = d.merge(subsetModelsWeights, left_on=['model'], right_on=['model'])
        d = d.drop(columns = ['weight'])
        d = d.rename(columns = {'renormWeight':'weight'})

        print("Sum of weights after renorm = {:f}".format(d.loc[:,['model','weight']].drop_duplicates().weight.sum()))

        return d

def removeHashValueFromValues(forecastsAndWeights):
    forecastsAndWeights['value'] = [ np.nan if x=='#VALUE!' else float(x) for x in forecastsAndWeights.value]
    return forecastsAndWeights

if __name__ == "__main__":

    forecasts = pd.read_csv('./forecasts/fluSightForecasts.csv')
    forecasts = formatBins(forecasts)

    weights   = pd.read_csv('./weights/adaptive-regularized-ensemble-constant.csv')

    mostRecentSurviellanceWeek = 201947
    forecastsAndWeights = forecasts.merge(weights, left_on=['model'],right_on=['component_model_id'])

    try:
        forecastsAndWeights = cap2MostRecentSurvWeek(forecastsAndWeights,mostRecentSurviellanceWeek)
    except NameError:
        mostRecentSurviellanceWeek = computeMostRecentSurvWeek()
        forecastsAndWeights = cap2MostRecentSurvWeek(forecastsAndWeights,mostRecentSurviellanceWeek)

    forecastsAndWeights = renormalizeWeightsForModelsThatDontSubmit(forecastsAndWeights)

    forecastsAndWeights = removeHashValueFromValues(forecastsAndWeights)
    forecastsAndWeights.value = forecastsAndWeights.value.astype(float)
    forecastsAndWeights['ensembleForecast'] = forecastsAndWeights.weight*forecastsAndWeights.value
    forecastsAndWeights = forecastsAndWeights.groupby(['location','target','type','unit','bin_start_incl','bin_end_notincl']).apply( lambda x: pd.Series({'ensembleForecast':x.ensembleForecast.sum()}) ).reset_index()

    ensembleForecast = formatEnsembleForecast(forecastsAndWeights)
    ensembleForecast = sortEnsembleForecast(ensembleForecast)

    EW,year,month,day = captureSubmissionInformation(mostRecentSurviellanceWeek)
    ensembleForecast.to_csv('./ensembleForecasts/EW{:02d}-KoT-adaptive-{:d}-{:d}-{:d}.csv'.format(EW,year,month,day),index=False)
    ensembleForecast.to_csv('./historicalEnsembleForecasts/EW{:02d}-KoT-adaptive-{:d}-{:d}-{:d}.csv'.format(EW,year,month,day),index=False)


import os
import sys
import numpy as np
import json
import io
from sklearn.model_selection import StratifiedKFold
from tqdm import tqdm

import paddle
from pgl.utils.data import Dataset,Dataloader

import pgl
from pgl.utils.logger import log

def random_split(dataset, split_ratio=0.7, seed=2019, shuffle=True):
    """random splitter"""
    np.random.seed(seed)
    indices = list(range(len(dataset)))
    np.random.shuffle(indices)
    trn_split = int(split_ratio * len(dataset))
    test_split = int(0.85 * len(dataset))

    train_idx, test_idx, valid_idx = indices[:trn_split], indices[trn_split:test_split], indices[test_split:]

    log.info("train_set : valid_set : test_set = %d : %d: %d" %
             (len(train_idx), len(valid_idx), len(test_idx)))
    return Subset(dataset, train_idx), Subset(dataset, valid_idx), Subset(dataset, test_idx)

class Subset(Dataset):
    """
    Subset of a dataset at specified indices.
    """

    def __init__(self, dataset, indices):
        self.dataset = dataset
        self.indices = indices

    def __getitem__(self, idx):
        """getitem"""
        return self.dataset[self.indices[idx]]

    def __len__(self):
        """len"""
        return len(self.indices)

class SIGNDataset(Dataset):

    """Dataset for Detecting Beneficial Feature Interactions for Recommender Systems (L0_SIGN)
    Adapted from https://github.com/ruizhang-ai/SIGN-Detecting-Beneficial-Feature-Interactions-for-Recommender-Systems/data/ml-tag/ml-tag.data.
    """

    def __init__(self,
                 data_path,
                 dataset_name,
                 pred_edges=1,
                 self_loop=True,
                 degree_as_nlabel=False):
        self.data_path = data_path
        self.dataset_name = dataset_name
        self.pred_edges = pred_edges
        self.self_loop = self_loop
        self.degree_as_nlabel = degree_as_nlabel

        self.graph_list = []
        self.glabel_list = []

        # global num
        self.num_graph = 0  # total graphs number
        self.num_feature = 0
        self.n = 0  # total nodes number
        self.m = 0  # total edges number

        self._load_data()

    def __len__(self):
        """return the number of graphs"""
        return len(self.graph_list)

    def __getitem__(self, idx):
        """getitem"""
        return self.graph_list[idx], self.glabel_list[idx]

    def read_data(self):
        # handle node and class
        filename = os.path.join(self.data_path, self.dataset_name,
                                "%s.data" % self.dataset_name)
        log.info("loading data from %s" % filename)
        node_list = []
        label = []
        max_node_index = 0
        data_num = 0
        with open(filename, 'r') as f:
            for line in f:
                data_num += 1
                data = line.split()
                # the first element is the label of the class
                label.append(float(data[0]))
                #the rest of the elements are the nodes
                int_list = [int(data[i]) for i in range(len(data))[1:]]
                node_list.append(int_list)
                if max_node_index < max(int_list):
                    max_node_index = max(int_list)

        if not self.pred_edges:
            # pass
            edge_list = []
            sr_list = []    #sender_receiver_list, containing node index
            for nodes in node_list:
                edge_l, sr_l = self.construct_full_edge_list(nodes)
                edge_list.append(edge_l)
                # sr_list.append(sr_l)

                # print('edge_l:',edge_l)
                # print('sr_l:',sr_l)
            # edge_list = [[[],[]] for _ in range(data_num)]
            # sr_list = []
            # # handle edges
            # with open(self.edgefile, 'r') as f:
            #     for line in f:
            #         edge_info = line.split()
            #         node_index = int(edge_info[0])
            #         edge_list[node_index][0].append(int(edge_info[1]))
            #         edge_list[node_index][1].append(int(edge_info[2]))
        else:
            edge_list = []
            sr_list = []    #sender_receiver_list, containing node index
            for nodes in node_list:
                edge_l, sr_l = self.construct_full_edge_list(nodes)
                edge_list.append(edge_l)
                sr_list.append(sr_l)

        # print(label[0:10])
        # print('==========='*10)
        label = self.construct_one_hot_label(label) # 去掉
        # print(label[0:10])

        return node_list, edge_list, label, sr_list, max_node_index + 1, data_num

    def construct_full_edge_list(self, nodes):
        num_node = len(nodes)
        edge_list = [[],[]]         #first for sender, second for receiver
        sender_receiver_list = []
        for i in range(num_node):
            for j in range(num_node)[i:]:
                edge_list[0].append(i)
                edge_list[1].append(j)
                sender_receiver_list.append([nodes[i],nodes[j]])

        return edge_list, sender_receiver_list

    def construct_one_hot_label(self, label):
        """Convert an iterable of indices to one-hot encoded labels."""
        nb_classes = int(max(label)) + 1
        targets = np.array(label, dtype=np.int32).reshape(-1)
        return np.eye(nb_classes)[targets]

    def _load_data(self):
        """Loads dataset
        """
        self.node, edge, label, self.sr_list, node_num, data_num = self.read_data()

        self.num_graph = len(self.node)
        self.num_feature = node_num

        for i in tqdm(range(self.num_graph)):
            # if int((i + 1) %  100000) == 0:
            #     log.info("processing graph %s" % (i + 1))

            graph = dict()
            edges = []
            num_edges = 0

            node_features = np.array(self.node[i],dtype='int32').reshape(len(self.node[i]),1)
            num_nodes = len(self.node[i])

            for u,v in zip(edge[i][0],edge[i][1]):
                u_v = (u,v)
                edges.append(u_v)

            num_edges = len(edges)

            self.glabel_list.append(label[i])

            if self.pred_edges:
                sr = self.sr_list[i]    #the sender receiver list, stored in edge_attr
            else:
                sr = []

            graph['node_attr'] = node_features
            graph['num_nodes'] = num_nodes
            graph['edge_attr'] = sr
            graph['edges'] = edges

            assert num_edges == len(edges)

            g = pgl.Graph(
                num_nodes=graph['num_nodes'],
                edges=graph['edges'],
                node_feat={'node_attr': graph['node_attr']},
                edge_feat={'edge_attr': graph['edge_attr']})

            self.graph_list.append(g)

            # update statistics of graphs
            self.n += graph['num_nodes']
            self.m += num_edges

        msg = "Finished loading data\n"
        msg += """
                num_graph:%d
                num_feature:%d
                nodes/graph:%d
                num_edges:%d
                """ %(
                    self.num_graph,
                    self.num_feature,
                    self.n/self.num_graph,
                    self.m
                )
        log.info(msg)

def collate_fn(batch_data):
    graphs = []
    labels = []
    for g, l in batch_data:
        graphs.append(g)
        labels.append(l)

    labels = np.array(labels, dtype="int64")

    return graphs, labels

if __name__ == "__main__":
    signdataset = SIGNDataset("./data/", "twitter", pred_edges=0)

    train_ds,val_ds,test_ds = random_split(signdataset)

    loader = Dataloader(
        train_ds,
        batch_size=3,
        shuffle=False,
        num_workers=1,
        collate_fn=collate_fn)
    cc = 0
    for batch in loader:
        g, label = batch
        # g = pgl.Graph.batch(g).tensor()

        print(label)
        print(g.num_graph)
        print('====='*20)
        # print(g.edges)
        print(g.graph_node_id)

        print(g.node_feat['node_attr'])
        print(g.edge_feat['edge_attr'])

        # for data in g:
        #     print(data.edges)

        # for data in g:
        #     print(data.node_feat['node_attr'])

        # for data in g:
        #     print(data.edge_feat['edge_attr'])

        cc += 1
        if cc == 2:
            break


"""add_is_synchronization_compatible_product
"""
from alembic import op
import sqlalchemy as sa

revision = "9e5ae228e4ab"
down_revision = "6a3db6c2a7b1"
branch_labels = None
depends_on = None

def upgrade():
    op.add_column(
        "product",
        sa.Column("isSynchronizationCompatible", sa.Boolean(), server_default=sa.text("true"), nullable=False),
    )

def downgrade():
    op.drop_column("product", "isSynchronizationCompatible")


#from key.key import *

import urllib
import json
import requests
import os

key = os.environ['WUNDERGROUND_KEY']

class CurrentWeather:
    def __init__(self, r, zip):
        self.zip = zip
        self.r = requests.get('http://api.wunderground.com/api/{}/geolookup/conditions/q/{}.json'.format(key, self.zip))

    def get(self):
        parsed_json = self.r.json()
        city = parsed_json['location']['city']
        state = parsed_json['location']['state']
        weather = parsed_json['current_observation']['weather']
        temperature_string = parsed_json['current_observation']['temperature_string']
        feelslike_string = parsed_json['current_observation']['feelslike_string']
        return print('Weather in ' + city + ', ' + state + ': ' + weather.lower() + '. The temperature is ' + temperature_string + ' but it feels like ' + feelslike_string + '.')

class TenDayWeather:
    def __init__(self, r, zip):
        self.zip = zip
        self.r = requests.get('http://api.wunderground.com/api/{}/geolookup/forecast10day/q/{}.json'.format(key, self.zip))

    def get(self):
        parsed_json = self.r.json()
        #print(json.dumps(parsed_json, indent=4, sort_keys=True))
        for day in parsed_json['forecast']['txt_forecast']['forecastday']:
            print("On "+day['title']+ " the forecast is "+day['fcttext'])

class SunRiseWeather:
    def __init__(self, r, zip):
        self.zip = zip
        self.r = requests.get('http://api.wunderground.com/api/{}/geolookup/astronomy/q/{}.json'.format(key, self.zip))

    def get(self):
        parsed_json = self.r.json()
        sunrise = parsed_json['sun_phase']['sunrise']['hour']+':'+parsed_json['sun_phase']['sunrise']['minute']
        sunset = parsed_json['sun_phase']['sunset']['hour']+':'+parsed_json['sun_phase']['sunset']['minute']
        print("Sunrise is at: " + sunrise + ", sunset is at: " + sunset)

class AlertsWeather:
    def __init__(self, r, zip):
        self.zip = zip
        self.r = requests.get('http://api.wunderground.com/api/{}/alerts/q/{}.json'.format(key, self.zip))

    def get(self):
        parsed_json = self.r.json()
        #print(json.dumps(parsed_json, indent=4, sort_keys=True))
        alert = parsed_json['alerts']
        if alert == []:
            print("There are no alerts for your area! Probably a good thing...")
        else:
            alert_type = parsed_json['alerts']['message']
            print("Oh no! The Weather Service says "+alert_type)

class HurricanesWeather:
    def __init__(self, r):
        self.r = requests.get('http://api.wunderground.com/api/{}/currenthurricane/view.json'.format(key))

    def get(self):
        parsed_json = self.r.json()
        print("There are currently "+str(len(parsed_json['currenthurricane']))+ " hurricanes: ")
        for hurricane in parsed_json['currenthurricane']:
        #print(json.dumps(hurricane, indent=4, sort_keys=True))
            name = hurricane['stormInfo']['stormName']
            print(name)

def get_zip():
    zip = input("What is your five digit zip code?\n")
    return zip

def main():
    option = input("""So what do you want to know?
    Enter 1 for current conditions;
    Enter 2 for ten day forecast;
    Enter 3 for sunrise and sunset times;
    Enter 4 for all current weather alerts;
    Enter 5 for all current tracked hurricanes.\n""" )
    if option == '1':
        new = CurrentWeather(key, get_zip())
        new.get()
    elif option == '2':
        new = TenDayWeather(key, get_zip())
        new.get()
    elif option == '3':
        new = SunRiseWeather(key, get_zip())
        new.get()
    elif option == '4':
        new = AlertsWeather(key, get_zip())
        new.get()
    else:
        new = HurricanesWeather(key)
        new.get()
main()


from __future__ import annotations

import json
import logging

from globals import api
from globals import con

def main():
    logger = logging.getLogger(__name__)
#    match = api.get_match(7207632145)
#    json.dump(match, open("7207632145.json", "w", encoding="utf-8"))
    player_matches = api.get_player_matches(293437752, take=50)
    json.dump(
        player_matches, open(
            'player_293437752.json', 'w', encoding='utf-8',
        ),
    )

    cur = con.cursor()
    cur.execute("""
                CREATE TABLE IF NOT EXISTS
                matches(id INT NOT NULL PRIMARY KEY, body TEXT)
                """)
    cur.execute("""
                CREATE TABLE IF NOT EXISTS
                match (id INT NOT NULL PRIMARY KEY, body TEXT)
                """)
    insert_matches = 'insert into matches(id, body) values(?, ?)'
    insert_match = 'insert into match(id, body) values(?, ?)'
    for match in player_matches:
        match_detail = ''
        match_detail = api.get_match(match['id'])
        try:
            con.execute(insert_match, (match['id'], json.dumps(match_detail)))
            con.execute(insert_matches, (match['id'], json.dumps(match)))
        except Exception as e:
            logger.error(f"Match ID: {match['id']}")
            logger.error(f"Sql error: {''.join(e.args)}")
            logger.error(f'Exception class: {e.__class__}')
            continue
    con.commit()

if __name__ == '__main__':
    main()


#!/usr/bin/env python

import sys
import os
from os.path import dirname, abspath
from optparse import OptionParser

from django.conf import settings

# For convenience configure settings if they are not pre-configured or if we
# haven't been provided settings to use by environment variable.
if not settings.configured and not os.environ.get('DJANGO_SETTINGS_MODULE'):
    settings.configure(
        DATABASES={
            'default': {
                'ENGINE': 'django.db.backends.sqlite3',
            }
        },
        AUTHENTICATION_BACKENDS = ("secure_login.backends.SecureLoginBackend", ),
        INSTALLED_APPS=[
            'django.contrib.admin',
            'django.contrib.auth',
            'django.contrib.contenttypes',
            'django.contrib.sessions',
            'django.contrib.messages',
            'django.contrib.staticfiles',
            'secure_login',
        ],
        STATIC_URL = "/static/",
    )

from django.test.simple import DjangoTestSuiteRunner

def runtests(*test_args, **kwargs):
    if not test_args:
        test_args = ['secure_login']
    parent = dirname(abspath(__file__))
    sys.path.insert(0, parent)
    test_runner = DjangoTestSuiteRunner(
        verbosity=kwargs.get('verbosity', 1),
        interactive=kwargs.get('interactive', False),
        failfast=kwargs.get('failfast'))
    failures = test_runner.run_tests(test_args)
    sys.exit(failures)

if __name__ == '__main__':
    parser = OptionParser()
    parser.add_option('--failfast', action='store_true', default=False, dest='failfast')
    (options, args) = parser.parse_args()
    runtests(failfast=options.failfast, *args)


from pydub import AudioSegment
import numpy, wave
import matplotlib.pyplot as plt
import numpy as np
import os
from itertools import groupby

from utils.codec import Codec
from utils.resampler import Resampler

def _load_wave(filename):
    # 根据文件名读取音频文件
    try:
        with wave.open(filename, "rb") as wf:
            print(wf.getparams())
            nchannels = wf.getparams().nchannels
            sampwidth = wf.getparams().sampwidth
            framerate = wf.getparams().framerate
            nframes = wf.getparams().nframes
            bytes_buffer = wf.readframes(nframes)  # 一次性读取所有frame

        audio_clip = Codec.decode_bytes_to_audio(bytes_buffer, nchannels,
                                                 sampwidth * 8)
        # if isinstance(audio_clip, tuple):
        #     audio_clip = Resampler.resample(audio_clip[1], framerate, 16000)
        # else:
        #     audio_clip = Resampler.resample(audio_clip, framerate, 16000)
        return audio_clip
    except:
        raise TypeError("Can't read wave file!")

def _find_burr(frames):
    abs_of_frames = np.abs(frames)
    mean_pow = np.mean(abs_of_frames)
    index_list = []
    for i, value in enumerate(frames):
        if value < -mean_pow * 7:
            index_list.append(i)
    return _group_continous_number(index_list)

def _group_continous_number(data):
    re = []
    fun = lambda x: x[1]-x[0]
    for k, g in groupby(enumerate(data), fun):
        l1 = [j for i, j in g]  # 连续数字的列表
        if len(l1) > 10:
            re.append(min(l1))
    return re

def run():
    # audio = AudioSegment.from_file("./offer_record.m4a", "m4a")
    # audio.export("./offer_record.wav", format="wav")
    wave_record = _load_wave("./tests/noise_record.wav")

    plt.figure(1)
    plt.scatter(np.linspace(0, len(wave_record)/16000, num=len(wave_record)),wave_record, s=0.2)
    plt.title("Record")

    print(_find_burr(wave_record))

    plt.show()

if __name__ == "__main__":
    run()


from django.http import JsonResponse
from django.shortcuts import render
from rest_framework import viewsets
from rest_framework.decorators import action
from rest_framework.request import Request
from rest_framework.response import Response

from .models import File, ExpertSystem
from .serializers import FileSerializer
# Create your views here.

class FileView(viewsets.ModelViewSet):
    queryset = File.objects.all().order_by('id')
    serializer_class = FileSerializer

    @staticmethod
    def list(request):
        files = File.objects.all().order_by('id')
        response = FileSerializer(files, many=True).data
        return Response(response)

    @staticmethod
    def create(request):
        print(request.FILES)
        print(request.data)
        name = request.data['name']
        file = request.data['file']
        if len(name) > 3 and file is not None:
            File.objects.create(name=name, file=file)
        files = File.objects.all().order_by('id')
        response = FileSerializer(files, many=True).data
        return Response(response)

    @action(methods=['get'], detail=True)
    def parse_file(self, request, pk):
        print(pk)
        selected_file = FileSerializer(File.objects.get(id=pk)).data
        print(selected_file['file'])
        es = ExpertSystem
        response = es.open_file(es, selected_file['file'])

        return Response(response)

    @action(methods=['post'], detail=True)
    def check_fact(self, request, pk):
        facts = request.data['facts']
        initial_facts = request.data['initial_facts']
        target_fact_id = request.data['target_fact_id']
        for f in facts:
            if f['id'] == target_fact_id:
                target_fact = f

        es_facts = []
        es_rules = []
        es_questions = []
        es = ExpertSystem(es_facts, es_rules, es_questions)
        selected_file = FileSerializer(File.objects.get(id=pk)).data
        es.open_file(selected_file['file'])
        response = es.check_fact(initial_facts, target_fact)

        return Response(response)


import config
import spacy
from bson.json_util import dumps
from constants import actions
from Templates import pln
from spade.agent import Agent
from spade.message import Message
from spade.behaviour import PeriodicBehaviour, OneShotBehaviour
from spade.template import Template
from spacy.lang.es.stop_words import STOP_WORDS

class RecvActionMainBehav(PeriodicBehaviour):
  async def run(self):
    msg_received = await self.receive()
    if msg_received and msg_received.get_metadata("action") == actions.EXTRACT_NAME:
      # Extraer nombre desde el texto
      res = pln.extract_name(msg_received.body)

      reply_msg = Message(to=config.AGENT_MAIN_USER)
      reply_msg.set_metadata("action", actions.EXTRACT_NAME)
      reply_msg.body = dumps({
        "found": res["found"],
        "body": res["name"]
      })
      await self.send(reply_msg)
    elif msg_received and msg_received.get_metadata("action") == actions.EXTRACT_REQUIREMENTS:
      # Extraer requerimientos desde el texto
      result = pln.extract_requirements(msg_received.body)
      reply_msg = Message(to=config.AGENT_MAIN_USER)
      reply_msg.set_metadata("action", actions.EXTRACT_REQUIREMENTS)
      reply_msg.body = dumps(result)
      await self.send(reply_msg)

class LangAgent(Agent):
  async def setup(self):
    print("Hola!. Soy el agente encargado del Lenguaje Natural. Mi ID es \"{}\"".format(str(self.jid)))

    recv_action_main_behav = RecvActionMainBehav(period=0.1)
    recv_action_main_behav_template = Template()
    recv_action_main_behav_template.to = config.AGENT_LANG_USER
    recv_action_main_behav_template.sender = config.AGENT_MAIN_USER
    self.add_behaviour(recv_action_main_behav)

  def stop(self):
    print("Deteniendo agente \"{}\"".format(str(self.jid)))
    return Agent.stop(self)


from aiogram import types, Dispatcher
from create_bot import dp, bot
import Livenshtein as liv
from keyboards import kb_client_inline_full, kb_necessity
from data_base import sqlite_db

'''Вывод ответа на вопрос пользователя'''

async def main_function_answer(message: types.Message):
    number = 0
    popular_answer = None
    s = liv.answer_db(message.text)
    popular_answer = liv.pop_answer(s)
    await bot.send_message(message.from_user.id, '--------------------\n\nНаиболее подходящий '
                                                 'ответ:\n\n--------------------')
    await bot.send_message(message.from_user.id, popular_answer[1])
    await bot.send_message(message.from_user.id, '--------------------\n\nПомог ли вам этот '
                                                 'ответ?\n\n--------------------', reply_markup=kb_necessity)
    await sqlite_db.add_quest_answer(message.from_user.id, message.text, popular_answer[1])
    await bot.send_message(message.from_user.id, '--------------------\n\nДругие похожие ответы:'
                                                 '\n\n--------------------')
    for answer in s:
        number += 1
        if number == 1:
            continue
        await bot.send_message(message.from_user.id, f'--------------------\nВопрос: {answer[0]}\n\nОтвет: {answer[1]}\n--------------------')

'''Вывод вопроса о полезности вопроса'''

@dp.callback_query_handler(lambda x: x.data and x.data.startswith('answer_'))
async def answer_necessity(callback):
    flag = callback.data.replace('answer_', '')
    await sqlite_db.add_necessity(callback.from_user.id, flag)
    await bot.send_message(callback.from_user.id, '--------------------\n\nВам помог ответ? Если нет, можете '
                                                  'попробовать сформулировать вопрос по другому или подать запрос на '
                                                  'связь со специалистом\n\n--------------------',
                           reply_markup=kb_client_inline_full)

def register_handlers_other(dp: Dispatcher):
    dp.register_message_handler(main_function_answer)


import collections, exceptions

from  MolarisTools.Units     import HARTREE_BOHR_TO_KCAL_MOL_ANGSTROM, HARTREE_TO_KCAL_MOL
from  MolarisTools.Utilities import TokenizeLine

Atom        = collections.namedtuple ("Atom"     , "symbol  x  y  z  charge")
Force       = collections.namedtuple ("Force"    , "x  y  z")

# . Convert from atomic units to Molaris units?
_DEFAULT_CONVERT_UNITS     = True

# . Multiply components of gradients by -1
_DEFAULT_REVERSE_GRADIENTS = False

class PCgradFile (object):
    """A class to read a file containing forces on point charges."""

    def __init__ (self, filename="run.pcgrad", reverse=_DEFAULT_REVERSE_GRADIENTS, convert=_DEFAULT_CONVERT_UNITS):
        """Constructor."""
        self.inputfile = filename
        self._Parse (reverse=reverse, convert=convert)

    def _Parse (self, reverse, convert):
        lines = open (self.inputfile)
        # . Get the number of atoms
        line   = next (lines)
        natoms = int (line)
        # . Read forces
        forces = []
        for i in range (natoms):
            line   = next (lines)
            gx, gy, gz = TokenizeLine (line, converters=[float, float, float])
            if reverse:
                gx, gy, gz = (-gx, -gy, -gz)
            if convert:
                gx, gy, gz = (gx * HARTREE_BOHR_TO_KCAL_MOL_ANGSTROM, gy * HARTREE_BOHR_TO_KCAL_MOL_ANGSTROM, gz * HARTREE_BOHR_TO_KCAL_MOL_ANGSTROM)
            force = Force (
                x   =   gx  ,
                y   =   gy  ,
                z   =   gz  ,
                )
            forces.append (force)
        self.forces = forces
        # . Close the file
        lines.close ()

#-------------------------------------------------------------------------------
class EngradFile (object):
    """A class to read a file containing forces on quantum atoms."""

    def __init__ (self, filename="run.engrad", reverse=_DEFAULT_REVERSE_GRADIENTS, convert=_DEFAULT_CONVERT_UNITS):
        """Constructor."""
        self.inputfile = filename
        self._Parse (reverse=reverse, convert=convert)

    def _Parse (self, reverse, convert):
        lines = open (self.inputfile)
        for i in range (3):
            next (lines)
        # . Get the number of atoms
        line   = next (lines)
        natoms = int (line)
        # . Skip a few lines
        for i in range (7):
            next (lines)
        # . Read forces
        forces = []
        for i in range (natoms):
            templ = []
            for j in range (3):
                line = next (lines)
                component = float (line)
                if reverse:
                    component = -component
                if convert:
                    component =  component * HARTREE_BOHR_TO_KCAL_MOL_ANGSTROM
                templ.append (component)
            force = Force (
                x   =   templ[0]    ,
                y   =   templ[1]    ,
                z   =   templ[2]    ,
                )
            forces.append (force)
        self.forces = forces
        # . Close the file
        lines.close ()

#-------------------------------------------------------------------------------
class ORCAOutputFile (object):
    """A class to read an ORCA output file."""

    def __init__ (self, filename="run.out", reverse=_DEFAULT_REVERSE_GRADIENTS, convert=_DEFAULT_CONVERT_UNITS):
        """Constructor."""
        self.inputfile = filename
        self._Parse (reverse=reverse, convert=convert)

    def _Parse (self, reverse, convert):
        lines = open (self.inputfile)
        # . Assume the job is failed until finding a "TERMINATED NORMALLY" statement
        jobOK = False
        try:
            while True:
                line = next (lines)
                # . Get coordinates of QM atoms
                # ---------------------------------
                # CARTESIAN COORDINATES (ANGSTROEM)
                # ---------------------------------
                #   C      5.663910    4.221157   -1.234141
                #   H      5.808442    3.140412   -1.242145
                # (...)
                if line.startswith ("CARTESIAN COORDINATES (ANGSTROEM)"):
                    next (lines)
                    line     = next (lines)
                    geometry = []
                    while line != "\n":
                        tokens = TokenizeLine (line, converters=[None, float, float, float])
                        label, x, y, z = tokens
                        atom   = (label, x, y, z)
                        geometry.append (atom)
                        line   = next (lines)

                # . Get charges on QM atoms
                # -----------------------
                # MULLIKEN ATOMIC CHARGES
                # -----------------------
                #    0 C :   -0.520010
                #    1 H :    0.271953
                # (...)
                # Sum of atomic charges:   -0.0000000
                if line.startswith ("MULLIKEN ATOMIC CHARGES"):
                    next (lines)
                    line    = next (lines)
                    charges = []
                    while not line.startswith ("Sum of atomic charges:"):
                        tokens = TokenizeLine (line, separator=":", converters=[None, float])
                        charge = tokens[1]
                        charges.append (charge)
                        line   = next (lines)
                    self.charges = charges
                    # . Construct the final list of atoms with charges
                    atoms   = []
                    for (label, x, y, z), charge in zip (geometry, charges):
                        atom   = Atom (
                            symbol  =   label   ,
                            x       =   x       ,
                            y       =   y       ,
                            z       =   z       ,
                            charge  =   charge  ,
                            )
                        atoms.append (atom)
                    self.atoms = atoms

                # . Get gradients on QM atoms
                # ------------------
                # CARTESIAN GRADIENT
                # ------------------
                #
                #    1   C   :   -0.017273415    0.000431161    0.011902545
                #    2   H   :    0.011246801   -0.004065387   -0.003492146
                # (...)
                elif line.startswith ("CARTESIAN GRADIENT"):
                    for i in range (2):
                        next (lines)
                    line   = next (lines)
                    forces = []
                    while line != "\n":
                        tokens = TokenizeLine (line, converters=[int, None, None, float, float, float])
                        gx, gy, gz = tokens[3:6]
                        if reverse:
                            gx, gy, gz = (-gx, -gy, -gz)
                        if convert:
                            gx, gy, gz = (gx * HARTREE_BOHR_TO_KCAL_MOL_ANGSTROM, gy * HARTREE_BOHR_TO_KCAL_MOL_ANGSTROM, gz * HARTREE_BOHR_TO_KCAL_MOL_ANGSTROM)
                        force  = Force (
                            x   =   gx  ,
                            y   =   gy  ,
                            z   =   gz  ,
                            )
                        forces.append (force)
                        line   = next (lines)
                    self.forces = forces

                # . Get the final energy
                # FINAL SINGLE POINT ENERGY      -263.834308915009
                elif line.startswith ("FINAL SINGLE POINT ENERGY"):
                    tokens = TokenizeLine (line, converters=[float, ], reverse=True)
                    if convert:
                        self.Efinal = tokens[-1] * HARTREE_TO_KCAL_MOL
                    else:
                        self.Efinal = tokens[-1]

                # . Check for a failed job
                elif line.count ("****ORCA TERMINATED NORMALLY****"):
                    jobOK = True
        except StopIteration:
            pass
        # . Close the file
        lines.close ()
        # . Check for a failed job
        if not jobOK:
            raise exceptions.StandardError ("Job %s did not end normally." % self.inputfile)

    @property
    def natoms (self):
        if hasattr (self, "atoms"):
            return len (self.atoms)
        return 0

#===============================================================================
# . Main program
#===============================================================================
if __name__ == "__main__": pass


import os
from typing import TYPE_CHECKING

import numpy as np
from geopandas import read_file
from shapely.geometry import Point
from starlette.applications import Starlette
from starlette.responses import JSONResponse
from starlette.routing import Route

from src.config import BASE_DIR
from src.exceptions import NotFoundError, ValidationError
from src.representatives import get_federal_representatives

if TYPE_CHECKING:
    from starlette.requests import Request
# Only 16-25kb of memory
GEOJSON = read_file(BASE_DIR / "districts.geojson")

def get_federal_district(index: int) -> str:
    # Possible to have TypeError if non-voting district
    # since their GeoJSON isn't valid?
    return GEOJSON.values[index][0][0]

# noinspection PyBroadException
async def find_district(request: "Request"):
    try:
        data: dict = await request.json()
        point = Point(float(data["longitude"]), float(data["latitude"]))
        index = np.where(GEOJSON.contains(point))[0]
        return JSONResponse({"district": get_federal_district(index)})
    except IndexError:
        return JSONResponse({"error": "could not find district"}, status_code=400)
    except TypeError:
        return JSONResponse(
            {
                "error": "you live in a non voting district. We're working on fixing that as soon as possible"
            },
            status_code=400,
        )
    except Exception:
        return JSONResponse({"error": "could not parse json"}, status_code=400)

# noinspection PyBroadException
async def find_representatives(request: "Request"):
    """Gets user's current senators and district representative"""
    try:
        data: dict = await request.json()
        return JSONResponse(
            get_federal_representatives(
                data["state"],
                data.get("district"),
                all_house=data.get("all_house"),
                all_house_regardless=data.get("all_house_regardless"),
            )
        )
    except (NotFoundError, ValidationError) as e:
        return JSONResponse({"error": e}, status_code=400)
    except Exception:
        return JSONResponse({"error": "could not parse json"}, status_code=400)

app = Starlette(  # I'm too lazy to setup python-dotenv...
    debug=os.environ.get("DEBUG") != "False",
    routes=[
        Route("/find-district", find_district),
        Route("/find-representative", find_representatives),
    ],
)


import concurrent.futures
import multiprocessing
from typing import List

from services.service_mesh.src.oci_cli_service_mesh.service_mesh_troubleshoot.aggregators.aggregation_orchestrator \
    import AggregationOrchestrator
from services.service_mesh.src.oci_cli_service_mesh.service_mesh_troubleshoot.task_request import TaskRequest, \
    TaskRequestType
from services.service_mesh.src.oci_cli_service_mesh.service_mesh_troubleshoot.tasks.common_worker import CommonTask
from services.service_mesh.src.oci_cli_service_mesh.service_mesh_troubleshoot.tasks.csv_worker import CsvTask
from services.service_mesh.src.oci_cli_service_mesh.service_mesh_troubleshoot.tasks.file_worker import FileTask
from services.service_mesh.src.oci_cli_service_mesh.service_mesh_troubleshoot.tasks.mesh_report_worker import \
    MeshReportTask
from services.service_mesh.src.oci_cli_service_mesh.service_mesh_troubleshoot.tasks.mesh_resources_worker import \
    MeshResourcesTask
from services.service_mesh.src.oci_cli_service_mesh.service_mesh_troubleshoot.tasks.namespaces_worker import \
    NamespacesTask
from services.service_mesh.src.oci_cli_service_mesh.service_mesh_troubleshoot.tasks.oci_cli_version_worker import \
    OciCliVersionTask
from services.service_mesh.src.oci_cli_service_mesh.service_mesh_troubleshoot.tasks.vdb_worker import \
    VdbTask
from services.service_mesh.src.oci_cli_service_mesh.service_mesh_troubleshoot.tasks.igd_worker import IgdTask
from services.service_mesh.src.oci_cli_service_mesh.service_mesh_troubleshoot.tasks.pod_worker import PodTask
from services.service_mesh.src.oci_cli_service_mesh.service_mesh_troubleshoot.tasks.resource_worker import \
    ResourceTask
from services.service_mesh.src.oci_cli_service_mesh.service_mesh_troubleshoot.tasks.validate_crds_worker import ValidateCrdsTask
from services.service_mesh.src.oci_cli_service_mesh.service_mesh_troubleshoot.tasks.webhooks_worker import WebhooksTask
from services.service_mesh.src.oci_cli_service_mesh.service_mesh_troubleshoot.tasks.services_worker import CheckServicesTask

class TaskOrchestrator():
    def __init__(self, ocid, k8s_context=None, thread_pool_size=None):
        self.ocid = ocid
        # Default thread pool size is 25
        self.thread_pool_size = 25 if thread_pool_size is None else thread_pool_size
        self.k8s_context = k8s_context
        self.task_processor_dict = {
            TaskRequestType.POD_PROCESSOR: PodTask(self.k8s_context),
            TaskRequestType.RESOURCE_PROCESSOR: ResourceTask(self.k8s_context),
            TaskRequestType.VDB_PROCESSOR: VdbTask(self.k8s_context),
            TaskRequestType.IGD_PROCESSOR: IgdTask(self.k8s_context),
            TaskRequestType.COMMON_PROCESSOR: CommonTask(self.k8s_context),
            TaskRequestType.NAMESPACES_PROCESSOR: NamespacesTask(self.k8s_context),
            TaskRequestType.OCI_CLI_VERSION_PROCESSOR: OciCliVersionTask(self.k8s_context),
            TaskRequestType.VALIDATE_CRDS_PROCESSOR: ValidateCrdsTask(self.k8s_context),
            TaskRequestType.WEBHOOKS_PROCESSOR: WebhooksTask(self.k8s_context),
            TaskRequestType.SERVICES_PROCESSOR: CheckServicesTask(self.k8s_context),
            TaskRequestType.FILE_PROCESSOR: FileTask(self.k8s_context),
            TaskRequestType.CSV_PROCESSOR: CsvTask(self.k8s_context),
            TaskRequestType.MESH_REPORT_PROCESSOR: MeshReportTask(self.k8s_context),
            TaskRequestType.MESH_RESOURCES_PROCESSOR: MeshResourcesTask(self.k8s_context),
        }

    def process_task(self, task):
        task_worker = self.task_processor_dict[task.task_request_type]
        return task_worker.execute(task, self.output_events, self.tasks_sm)

    def process_tasks(self, tasks: List[TaskRequest]):
        futures_list, done, not_done = [], [], []
        m = multiprocessing.Manager()
        self.tasks_sm, self.output_events = m.Queue(), m.Queue()
        self.tasks_sm.put(tasks.pop())
        while not self.tasks_sm.empty() or len(not_done) > 0:
            done, not_done = concurrent.futures.wait(futures_list)
            task_list = []
            while not self.tasks_sm.empty():
                task_list.append(self.tasks_sm.get())
            executor = concurrent.futures.ThreadPoolExecutor(self.thread_pool_size)
            futures = [executor.submit(self.process_task, item) for item in task_list]
            concurrent.futures.wait(futures)

        aggregation_orchestrator = AggregationOrchestrator(self.ocid)
        aggregation_orchestrator.aggregate_events(self.output_events)


#!/usr/bin/env python
# -*- encoding: utf-8 -*-

import time
import unittest
import HTMLTestRunner
from utils.config import report_path,testcase_path,caselist_path
from utils.e_mail import Email
from utils.log import Logger

class Run_Suite():

    def __init__(self):
        self.caselist = []
        self.log = Logger().get_logger()

    def get_caselist(self):
        with open(caselist_path,'r') as fp:
            for line in fp.readlines():
                data = str(line)
                if data != '' and not data.startswith("#"):
                    self.caselist.append(data.replace("\n", ""))
        return  self.caselist

    def set_case_suite(self):
        test_suite = unittest.TestSuite()
        suite_module = []
        for case in  self.get_caselist():
            case_name = case.split("/")[-1]  # 通过split函数来将aaa/bbb分割字符串，-1取后面，0取前面
            discover = unittest.defaultTestLoader.discover(testcase_path,pattern = case_name + '.py',top_level_dir=None)
            suite_module.append(discover)#将discover存入suite_module元素组

        if len(suite_module) > 0:#判断suite_module元素组是否存在元素
            for suite in suite_module:#如果存在，循环取出元素组内容，命名为suite
                for case_name in suite:#从discover中取出case_name，使用addTest添加到测试集
                    test_suite.addTest(case_name)
        else:
            return None
        return test_suite#返回测试集

    def run_case_suite(self):
        try:
            suit = self.set_case_suite()#调用set_case_suite获取test_suite
            print(str(suit))
            if suit is not None:#判断test_suite是否为空
                now = time.strftime("%Y-%m-%d-%H %M %S", time.localtime(time.time()))
                filename = report_path + "\\" + now + "-" + "ResultReport.html"
                with open(filename, 'wb') as fp:
                    runner = HTMLTestRunner.HTMLTestRunner(stream=fp, title=u"测试报告",
                                                           description=u"测试用例执行的结果", verbosity=2)
                    runner.run(suit)

            else:
                print("没有要执行的测试用例.")
        except Exception as ex:
            print(str(ex))
            self.log.info(str(ex))

        finally:
            print("*********TEST END*********")
            self.log.info(str("*********TEST END*********"))

        e = Email()
        e.send_email()

if __name__ == "__main__":
    c = Run_Suite()
    data = c.run_case_suite()


import tkinter as tk
from tkinter import ttk
import threading
import keyboard
import time
from tkinter import messagebox

icon_path = 'C:\\Users\\Mr_Pryor\\Desktop\\Save Scummer\\bg3.ico'

class KeySenderApp:
    def __init__(self, root):
        self.root = root
        self.root.title("BG3: Save Scummer")  # Change the window title here

        # Set the background color to #313338
        self.root.configure(bg='#313338')

        # Calculate the center position of the screen
        screen_width = root.winfo_screenwidth()
        screen_height = root.winfo_screenheight()
        x = (screen_width - 300) // 2  # 300 is the width of the window
        y = (screen_height - 250) // 2  # 250 is the height of the window

        # Set the window size and position
        self.root.geometry("300x200+{}+{}".format(x, y))

        # Set the window icon
        self.root.iconbitmap(icon_path)

        # Create a custom style to match Discord's color scheme
        self.style = ttk.Style()
        self.style.configure('TLabel', foreground='#949B9A', background='#313338')  # Change label text color to #949B9A and background to #313338
        self.style.configure('TEntry', foreground='black', background='#2B2D31')  # Change text color to black and entry background to #2B2D31
        self.style.configure('TCombobox', foreground='black', background='#2C2F33')  # Change text color to black and combobox background to #2C2F33
        self.style.configure('TButton', foreground='black', background='#B5BAC1')  # Change text color to black and button background to #B5BAC1
        self.style.configure('TFrame', background='#1E1F22')  # Change the frame background color to #1E1F22
        self.style.configure('TNotebook', background='#1E1F22')  # Change the notebook background color to #1E1F22

        self.key_label = ttk.Label(root, text="Key", style='TLabel')
        self.key_label.pack()

        # Set the default key to "F5"
        self.key_entry = ttk.Entry(root, style='TEntry')
        self.key_entry.insert(0, "F5")
        self.key_entry.pack()

        self.key_set = False  # Flag to check if the key is set

        self.delay_label = ttk.Label(root, text="Delay", style='TLabel')
        self.delay_label.pack()

        # Set the default delay to "5"
        self.delay_var = tk.StringVar()
        vcmd = (self.root.register(self.validate_delay_input), "%P")
        self.delay_entry = ttk.Entry(root, style='TEntry', textvariable=self.delay_var, validate="key", validatecommand=vcmd)
        self.delay_entry.insert(0, "5")
        self.delay_entry.pack()

        # Set the default delay units to "Minutes"
        self.delay_units = tk.StringVar()
        self.delay_units.set("Minutes")

        self.delay_units_label = ttk.Label(root, text="Delay Units", style='TLabel')
        self.delay_units_label.pack()

        self.delay_units_combobox = ttk.Combobox(root, textvariable=self.delay_units, values=["Seconds", "Minutes"], state="readonly", style='TCombobox')
        self.delay_units_combobox.pack()

        self.countdown_label = ttk.Label(root, text="Countdown:", style='TLabel')
        self.countdown_label.pack()

        self.countdown_value = tk.StringVar()
        self.countdown_label_value = ttk.Label(root, textvariable=self.countdown_value, style='TLabel')
        self.countdown_label_value.pack()

        # Create a toggle button for start/stop
        self.start_button = ttk.Button(root, text="Start", command=self.toggle_countdown, style='TButton')
        self.start_button.pack()

        self.is_counting_down = False
        self.countdown_thread = None

        # Store the previous delay units
        self.prev_delay_units = "Minutes"

        # Counter for key presses
        self.key_press_count = 0
        self.key_press_label = ttk.Label(root, text="Saves: {}".format(self.key_press_count), style='TLabel')
        self.key_press_label.pack(side="bottom", anchor="se")

        # Bind a key press event to capture the key
        self.root.bind("<KeyPress>", self.capture_key_press)

    def validate_delay_input(self, input_text):
        # This function validates the input in the Delay entry field to allow only numbers and an empty string.
        return input_text == "" or input_text.isdigit()

    def toggle_countdown(self):
        if self.is_counting_down:
            self.stop_countdown()
        else:
            self.start_countdown()

    def start_countdown(self):
        if not self.is_counting_down:
            delay = float(self.delay_var.get())

            if delay <= 1:
                # Display an error message and return without starting the countdown
                messagebox.showerror("Error", "Delay must be greater than 1.")
                return

            self.is_counting_down = True
            self.start_button.config(text="Stop")
            if self.delay_units.get() == "Minutes":
                delay *= 60

            self.stop_countdown_event = threading.Event()

            def countdown(delay):
                original_delay = delay  # Store the original delay
                while not self.stop_countdown_event.is_set():
                    while delay > 0 and not self.stop_countdown_event.is_set():
                        self.countdown_value.set(f"{delay:.1f} seconds")
                        time.sleep(1)
                        delay -= 1
                    if not self.stop_countdown_event.is_set():
                        self.send_key()
                        delay = original_delay  # Reset the delay to the original value

                self.reset_countdown()

            self.countdown_thread = threading.Thread(target=countdown, args=(delay,))
            self.countdown_thread.start()

    def stop_countdown(self):
        if self.is_counting_down:
            self.stop_countdown_event.set()

    def send_key(self):
        key_to_send = self.key_entry.get()
        keyboard.press_and_release(key_to_send)
        self.key_press_count += 1
        self.key_press_label.config(text="Saves: {}".format(self.key_press_count))

    def reset_countdown(self):
        self.is_counting_down = False
        self.start_button.config(text="Start")
        self.countdown_value.set("")

        # Check if the countdown thread is not the current thread before attempting to join it
        if self.countdown_thread and self.countdown_thread != threading.current_thread():
            self.countdown_thread.join()

    def on_delay_units_change(self, *args):
        # Check if delay units have changed
        if self.delay_units.get() != self.prev_delay_units:
            self.prev_delay_units = self.delay_units.get()
            self.reset_countdown()  # Reset the countdown

    def capture_key_press(self, event):
        # This function captures the key press event and sets the key entry field to the pressed key.
        if not self.key_set:
            pressed_key = event.keysym
            self.key_entry.delete(0, tk.END)
            self.key_entry.insert(0, pressed_key)
            self.key_set = True

if __name__ == "__main__":
    root = tk.Tk()
    app = KeySenderApp(root)
    # Bind the delay units change event to the handler
    app.delay_units.trace_add("write", app.on_delay_units_change)
    root.mainloop()


'''
lxml--解析HTML
'''
from lxml import etree

text = '''

first item

first item

first item

first item

first item

'''
# 利用etree.HTML()把字符串解析成HTML文件
# 会自动补全html代码
html = etree.HTML(text)
s = etree.tostring(html)
print(s)

'''
lxml--XML文件读取
'''
html = etree.parse("./05--p05.xml")
print(type(html))
rst = etree.tostring(html, pretty_print=True)
print(rst)

'''
etree和XPath的配合使用
'''
html = etree.parse("./05--p05.xml")
print(type(html))

rst = html.xpath("//book")
print(type(rst))
print(rst)

rst = html.xpath("//book[@categary='cook']/price")
rst = rst[0]
print(type(rst))
print(rst.tag)
print(rst.text)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from refinery.units.blockwise import BlockTransformationBase

class rev(BlockTransformationBase):
    """
    The blocks of the input data are output in reverse order. If the length of
    the input data is not a multiple of the block size, the data is truncated.
    """
    def __init__(self, blocksize=1):
        super().__init__(blocksize=blocksize)

    def process(self, data):
        if self.bytestream:
            return data[::-1]
        else:
            rv = list(self.chunk(data, raw=True))[::-1]
            return self.rest(data) + self.unchunk(rv, raw=True)


import torch
import torch.nn as nn
import torch.optim as optim
import torch.optim.lr_scheduler as lr_scheduler
from torch.utils.data import DataLoader

import argparse
import rdkit
import math, random, sys, os
import numpy as np
from tqdm import tqdm

from fuseprop import *
from properties import get_scoring_function

qed_sa_func = lambda x: x[0] >= 0.5 and x[1] >= 0.5 and x[2] >= 0.6 and x[3] <= 4.0
normal_func = lambda x: min(x) >= 0.5

def remove_order(s):
    for x in range(15):
        s = s.replace(":%d]" % (x,), ":1]")
    return s

# Decode molecules
def decode_rationales(model, rationale_dataset):
    loader = DataLoader(rationale_dataset, batch_size=1, shuffle=False, num_workers=0, collate_fn=lambda x:x[0])
    model.eval()
    cand_mols = []
    with torch.no_grad():
        for init_smiles in tqdm(loader):
            final_smiles = model.decode(init_smiles)
            mols = [(x,y) for x,y in zip(init_smiles, final_smiles) if y and '.' not in y]
            mols = [(x,y) for x,y in mols if Chem.MolFromSmiles(y).HasSubstructMatch(Chem.MolFromSmiles(x))]
            cand_mols.extend(mols)
    return cand_mols

# Predict properties and filter
def property_filter(cand_mols, scoring_function, args):
    rationales, smiles_list = zip(*cand_mols)
    cand_props = scoring_function(smiles_list)
    new_data = []
    rationale_dist = {remove_order(r) : [0,0] for r in rationales}
    with open(args.save_dir + '/valid.' + str(epoch), 'w') as f:
        for (init_smiles, final_smiles), prop in zip(cand_mols, cand_props):
            rationale = remove_order(init_smiles)
            rationale_dist[rationale][1] += 1
            print(init_smiles, final_smiles, prop, file=f)
            if args.compare_func(prop):
                rationale_dist[rationale][0] += 1
                new_data.append( (init_smiles, final_smiles) )

    print('property filter: %d -> %d' % (len(cand_mols), len(new_data)))
    rationale_dist = {r : x / n for r,(x,n) in rationale_dist.items() if x / n >= args.alpha}
    return new_data, rationale_dist

if __name__ == "__main__":
    lg = rdkit.RDLogger.logger()
    lg.setLevel(rdkit.RDLogger.CRITICAL)

    parser = argparse.ArgumentParser()
    parser.add_argument('--rationale', required=True)
    parser.add_argument('--prop', required=True)
    parser.add_argument('--save_dir', required=True)
    parser.add_argument('--init_model', type=str)
    parser.add_argument('--load_epoch', type=int, default=-1)
    parser.add_argument('--atom_vocab', default=common_atom_vocab)

    parser.add_argument('--rnn_type', type=str, default='LSTM')
    parser.add_argument('--hidden_size', type=int, default=400)
    parser.add_argument('--embed_size', type=int, default=400)
    parser.add_argument('--batch_size', type=int, default=10)
    parser.add_argument('--decode_batch_size', type=int, default=20)
    parser.add_argument('--latent_size', type=int, default=20)
    parser.add_argument('--depth', type=int, default=10)
    parser.add_argument('--diter', type=int, default=3)

    parser.add_argument('--lr', type=float, default=5e-4)
    parser.add_argument('--clip_norm', type=float, default=20.0)
    parser.add_argument('--alpha', type=float, default=1.0)
    parser.add_argument('--beta', type=float, default=0.3)
    parser.add_argument('--num_decode', type=int, default=200)

    parser.add_argument('--epoch', type=int, default=1)
    parser.add_argument('--anneal_rate', type=float, default=1.0)
    parser.add_argument('--print_iter', type=int, default=50)

    args = parser.parse_args()
    print(args)

    if args.prop == "gsk3,jnk3,qed,sa" or args.prop == "jnk3,gsk3,qed,sa":
        args.compare_func = qed_sa_func
    else:
        args.compare_func = normal_func

    prop_funcs = [get_scoring_function(prop) for prop in args.prop.split(',')]
    scoring_function = lambda x : list( zip(*[func(x) for func in prop_funcs]) )

    with open(args.rationale) as f:
        rationales = [line.split()[1] for line in f]
        rationales = unique_rationales(rationales)
        rationale_dataset = SubgraphDataset(rationales, args.atom_vocab, args.decode_batch_size, args.num_decode)

    model = AtomVGNN(args).cuda()
    if args.load_epoch >= 0:
        path = os.path.join(args.save_dir, f"model.{args.load_epoch}")
        model.load_state_dict(torch.load(path)[1])
    else:
        model.load_state_dict(torch.load(args.init_model))

    print("Model #Params: %dK" % (sum([x.nelement() for x in model.parameters()]) / 1000,))

    optimizer = optim.Adam(model.parameters(), lr=args.lr)
    scheduler = lr_scheduler.ExponentialLR(optimizer, args.anneal_rate)

    param_norm = lambda m: math.sqrt(sum([p.norm().item() ** 2 for p in m.parameters()]))
    grad_norm = lambda m: math.sqrt(sum([p.grad.norm().item() ** 2 for p in m.parameters() if p.grad is not None]))

    for epoch in range(args.load_epoch + 1, args.epoch):
        print('epoch', epoch)
        cand_mols = decode_rationales(model, rationale_dataset)
        cand_mols, rationale_dist = property_filter(cand_mols, scoring_function, args)

        model_ckpt = (rationale_dist, model.state_dict())
        torch.save(model_ckpt, os.path.join(args.save_dir, f"model.{epoch}"))

        cand_mols = list(set(cand_mols))
        random.shuffle(cand_mols)

        # Update model
        dataset = MoleculeDataset(cand_mols, args.atom_vocab, args.batch_size)
        dataloader = DataLoader(dataset, batch_size=1, shuffle=True, num_workers=0, collate_fn=lambda x:x[0])
        model.train()

        meters = np.zeros(5)
        for total_step, batch in enumerate(dataset):
            if batch is None: continue

            model.zero_grad()
            loss, kl_div, wacc, tacc, sacc = model(*batch, beta=args.beta)
            loss.backward()
            nn.utils.clip_grad_norm_(model.parameters(), args.clip_norm)
            optimizer.step()

            meters = meters + np.array([kl_div, loss.item(), wacc * 100, tacc * 100, sacc * 100])

            if (total_step + 1) % args.print_iter == 0:
                meters /= args.print_iter
                print("[%d] Beta: %.3f, KL: %.2f, loss: %.3f, Word: %.2f, Topo: %.2f, Assm: %.2f, PNorm: %.2f, GNorm: %.2f" % (total_step + 1, args.beta, meters[0], meters[1], meters[2], meters[3], meters[4], param_norm(model), grad_norm(model)))
                sys.stdout.flush()
                meters *= 0

        scheduler.step()
        print("learning rate: %.6f" % scheduler.get_lr()[0])


import sys
import typing

__author__ = "Predeactor"
__authors__ = ["Predeactor <pro.julien.mauroy@gmail.com>"]
__version__ = "0.2.1"

if sys.version_info.minor > 9:
    _F = typing.ParamSpec("_F")
else:
    from typing_extensions import ParamSpec
    _F = ParamSpec("_F")  # type: ignore
_R = typing.TypeVar("_R")

def __ensure_target(target: typing.Any, property: typing.Optional[str]) -> typing.Any:
    # Ensure target is not None
    if not target:
        raise ValueError("Subject is None, can't lookup empty object")

    # Define what we must lookup (Target or property of target)
    final_target = getattr(target, property) if property else target
    if not final_target:
        raise ValueError("Could not obtain property in '{subject}'")

    return final_target

def metadata(
    key: str, value: typing.Any
) -> typing.Callable[[typing.Callable[_F, _R]], typing.Callable[_F, _R]]:
    def decorator(target: typing.Callable[_F, _R]) -> typing.Callable[_F, _R]:
        define_metadata(target, key, value, None)
        return target

    return decorator

def has_metadata(target: typing.Any, property: typing.Optional[str] = None) -> bool:
    final_target = __ensure_target(target, property)

    return (
        len(final_target.metadata) > 0 if hasattr(final_target, "metadata") else False
    )

def define_metadata(
    target: typing.Any,
    key: str,
    value: typing.Any,
    property: typing.Optional[str] = None,
):
    final_target = __ensure_target(target, property)

    if not hasattr(final_target, "metadata"):
        final_target.metadata = {}
    final_target.metadata[key] = value

def get_metadata(
    target: typing.Any, key: str, property: typing.Optional[str] = None
) -> typing.Optional[typing.Any]:
    final_target = __ensure_target(target, property)

    if not hasattr(final_target, "metadata"):
        raise ValueError(f"No metadata has been registered to '{target}'")

    # Get all metadata and see if the one we want is registered
    metadata = final_target.metadata

    if key not in metadata:
        raise ValueError(f"Metadata '{key}' is not set on object '{target}'")

    # And voilà!
    return metadata[key]

def list_metadata(
    target: typing.Any, property: typing.Optional[str] = None
) -> typing.Dict[str, typing.Any]:
    final_target = __ensure_target(target, property)

    return final_target.metadata

def delete_metadata(
    target: typing.Any, key: str, property: typing.Optional[str] = None
) -> None:
    final_target = __ensure_target(target, property)

    if not hasattr(final_target, "metadata"):
        return
    metadata = final_target.metadata

    if key in metadata:
        del metadata[key]


# -*- coding: utf8 -*-
try:
    __instance = unicode
except:
    __instance = str

import gzip
import os
import StringIO
import tempfile
import urllib2

# Some sites blocks default python User-agent
user_agent = 'Mozilla/5.0 (X11; Linux x86_64; rv:68.0) Gecko/20100101 Firefox/68.0'
headers = {'User-Agent': user_agent}

def download_file(url, referer=None):
    """ Download file at url and write it to a file, return the path to the file and the url """
    file, path = tempfile.mkstemp()
    file = os.fdopen(file, "w")
    content_type = None
    # Download url
    req = urllib2.Request(url, headers=headers)
    if referer is not None:
        req.add_header('referer', referer)
    try:
        response = urllib2.urlopen(req)
    except urllib2.HTTPError:
        return None, None
    else:
        dat = response.read()
        content_type = response.info().getheader('Content-Type')
        # Check if it is gzipped
        if dat[:2] == '\037\213':
            # Data is gzip encoded, decode it
            compressedstream = StringIO.StringIO(dat)
            gzipper = gzip.GzipFile(fileobj=compressedstream)
            extracted_data = gzipper.read()
            dat = extracted_data

        # Write it to a file
        file.write(dat)
        file.close()
        # return file path
        return path, content_type

class unicode_tr(__instance):
    CHARMAP = {
        "to_upper": {
            u"ı": u"I",
            u"i": u"İ",
            u"ğ": u"Ğ",
            u"ü": u"Ü",
            u"ş": u"Ş",
            u"ö": u"Ö",
            u"ç": u"Ç",
        },
        "to_lower": {
            u"I": u"ı",
            u"İ": u"i",
            u"Ğ": u"ğ",
            u"Ü": u"ü",
            u"Ş": u"ş",
            u"Ö": u"ö",
            u"Ç": u"ç",
        },
        "to_loweer": {
            u"ı": u"i",
            u"ğ": u"g",
            u"ü": u"u",
            u"ş": u"s",
            u"ö": u"o",
            u"ç": u"c",
            u"I": u"i",
            u"İ": u"i",
            u"Ğ": u"g",
            u"Ü": u"u",
            u"Ş": u"s",
            u"Ö": u"o",
            u"Ç": u"c",
        },
    }

    def lower(self):
        for key, value in self.CHARMAP.get("to_lower").items():
            self = self.replace(key, value)
        return self.lower()

    def upper(self):
        for key, value in self.CHARMAP.get("to_upper").items():
            self = self.replace(key, value)
        return self.upper()

    def loweer(self):
        for key, value in self.CHARMAP.get("to_loweer").items():
            self = self.replace(key, value)
        return self.lower()

    def capitalize(self):
        first, rest = self[0], self[1:]
        return unicode_tr(first).upper() + unicode_tr(rest).lower()

    def title(self):
        words = u've ile and with'
        return " ".join(
            map(lambda x: unicode_tr(x).lower() if unicode_tr(x).lower() in words else unicode_tr(x).capitalize(),
                self.split()))


import itertools
import logging
import numpy as np
import copy
import torch.utils.data
import torch

from detectron2.config import configurable
from detectron2.utils.logger import _log_api_usage

from detectron2.data.catalog import DatasetCatalog, MetadataCatalog
from detectron2.data.common import DatasetFromList, MapDataset
from detectron2.data.dataset_mapper import DatasetMapper
from detectron2.data.detection_utils import check_metadata_consistency
from detectron2.data.samplers import InferenceSampler, RepeatFactorTrainingSampler, TrainingSampler

from detectron2.data.build import trivial_batch_collator
from detectron2.data import (build_batch_data_loader,
                             print_instances_class_histogram,
                             load_proposals_into_dataset)
from detectron2.data.build import (filter_images_with_few_keypoints,
                                   filter_images_with_only_crowd_annotations)

"""
This file contains the default logic to build a dataloader for training or testing.
"""
__all__ = [
    "build_detection_train_loader",
    "build_detection_test_loader",
    "get_detection_dataset_dicts",
]

def get_detection_dataset_dicts(names, filter_empty=True, min_keypoints=0, proposal_files=None, cfg=None):
    """
    Load and prepare dataset dicts for instance detection/segmentation and semantic segmentation.

    Args:
        names (str or list[str]): a dataset name or a list of dataset names
        filter_empty (bool): whether to filter out images without instance annotations
        min_keypoints (int): filter out images with fewer keypoints than
            `min_keypoints`. Set to 0 to do nothing.
        proposal_files (list[str]): if given, a list of object proposal files
            that match each dataset in `names`.

    Returns:
        list[dict]: a list of dicts following the standard dataset dict format.
    """
    if isinstance(names, str):
        names = [names]
    assert len(names), names
    dataset_dicts = [DatasetCatalog.get(dataset_name)
                     for dataset_name in names]
    for dataset_name, dicts in zip(names, dataset_dicts):
        assert len(dicts), "Dataset '{}' is empty!".format(dataset_name)

    if proposal_files is not None:
        assert len(names) == len(proposal_files)
        # load precomputed proposals from proposal files
        dataset_dicts = [
            load_proposals_into_dataset(dataset_i_dicts, proposal_file)
            for dataset_i_dicts, proposal_file in zip(dataset_dicts, proposal_files)
        ]

    dataset_dicts = list(itertools.chain.from_iterable(dataset_dicts))

    has_instances = "annotations" in dataset_dicts[0]
    if filter_empty and has_instances:
        dataset_dicts = filter_images_with_only_crowd_annotations(
            dataset_dicts)
    if min_keypoints > 0 and has_instances:
        dataset_dicts = filter_images_with_few_keypoints(
            dataset_dicts, min_keypoints)

    d_name = names[0]
    # if 'voc_coco' in d_name:
    if 'train' in d_name:
        dataset_dicts = remove_unk_instances(cfg, dataset_dicts)
    elif 'test' in d_name:
        dataset_dicts = label_known_class_and_unknown(cfg, dataset_dicts)

    if has_instances:
        try:
            class_names = MetadataCatalog.get(names[0]).thing_classes
            check_metadata_consistency("thing_classes", names)
            print_instances_class_histogram(dataset_dicts, class_names)
        except AttributeError:  # class names are not available for this dataset
            pass

    assert len(dataset_dicts), "No valid data found in {}.".format(
        ",".join(names))
    return dataset_dicts

def remove_unk_instances(cfg, dataset_dicts):
    num_known_classes = cfg.MODEL.ROI_HEADS.NUM_KNOWN_CLASSES
    valid_classes = range(0, num_known_classes)

    logger = logging.getLogger(__name__)
    logger.info("Valid classes: " + str(valid_classes))
    logger.info("Removing unknown objects...")

    for entry in copy.copy(dataset_dicts):
        annos = entry["annotations"]
        for annotation in copy.copy(annos):
            if annotation["category_id"] not in valid_classes:
                annos.remove(annotation)
        if len(annos) == 0:
            dataset_dicts.remove(entry)

    return dataset_dicts

def label_known_class_and_unknown(cfg, dataset_dicts):
    num_known_classes = cfg.MODEL.ROI_HEADS.NUM_KNOWN_CLASSES
    total_num_class = cfg.MODEL.ROI_HEADS.NUM_CLASSES

    known_classes = range(0, num_known_classes)

    logger = logging.getLogger(__name__)
    logger.info("Known classes: " + str(known_classes))
    logger.info(
        "Labelling known instances the corresponding label, and unknown instances as unknown...")

    for entry in dataset_dicts:
        annos = entry["annotations"]
        for annotation in annos:
            if annotation["category_id"] not in known_classes:
                annotation["category_id"] = total_num_class - 1

    return dataset_dicts

def _train_loader_from_config(cfg, mapper=None, *, dataset=None, sampler=None):
    if dataset is None:
        dataset = get_detection_dataset_dicts(
            cfg.DATASETS.TRAIN,
            filter_empty=cfg.DATALOADER.FILTER_EMPTY_ANNOTATIONS,
            min_keypoints=cfg.MODEL.ROI_KEYPOINT_HEAD.MIN_KEYPOINTS_PER_IMAGE
            if cfg.MODEL.KEYPOINT_ON
            else 0,
            proposal_files=cfg.DATASETS.PROPOSAL_FILES_TRAIN if cfg.MODEL.LOAD_PROPOSALS else None,
            cfg=cfg
        )
        _log_api_usage("dataset." + cfg.DATASETS.TRAIN[0])

    if mapper is None:
        mapper = DatasetMapper(cfg, True)
    else:
        mapper = mapper(cfg, True)

    if sampler is None:
        sampler_name = cfg.DATALOADER.SAMPLER_TRAIN
        logger = logging.getLogger(__name__)
        logger.info("Using training sampler {}".format(sampler_name))
        if sampler_name == "TrainingSampler":
            sampler = TrainingSampler(len(dataset))
        elif sampler_name == "RepeatFactorTrainingSampler":
            repeat_factors = RepeatFactorTrainingSampler.repeat_factors_from_category_frequency(
                dataset, cfg.DATALOADER.REPEAT_THRESHOLD
            )
            sampler = RepeatFactorTrainingSampler(repeat_factors)
        else:
            raise ValueError(
                "Unknown training sampler: {}".format(sampler_name))

    return {
        "dataset": dataset,
        "sampler": sampler,
        "mapper": mapper,
        "total_batch_size": cfg.SOLVER.IMS_PER_BATCH,
        "aspect_ratio_grouping": cfg.DATALOADER.ASPECT_RATIO_GROUPING,
        "num_workers": cfg.DATALOADER.NUM_WORKERS,
    }

# TODO can allow dataset as an iterable or IterableDataset to make this function more general
@configurable(from_config=_train_loader_from_config)
def build_detection_train_loader(
    dataset, *, mapper, sampler=None, total_batch_size, aspect_ratio_grouping=True, num_workers=0
):
    """
    Build a dataloader for object detection with some default features.
    This interface is experimental.

    Args:
        dataset (list or torch.utils.data.Dataset): a list of dataset dicts,
            or a map-style pytorch dataset. They can be obtained by using
            :func:`DatasetCatalog.get` or :func:`get_detection_dataset_dicts`.
        mapper (callable): a callable which takes a sample (dict) from dataset and
            returns the format to be consumed by the model.
            When using cfg, the default choice is ``DatasetMapper(cfg, is_train=True)``.
        sampler (torch.utils.data.sampler.Sampler or None): a sampler that produces
            indices to be applied on ``dataset``. Default to :class:`TrainingSampler`,
            which coordinates an infinite random shuffle sequence across all workers.
        total_batch_size (int): total batch size across all workers. Batching
            simply puts data into a list.
        aspect_ratio_grouping (bool): whether to group images with similar
            aspect ratio for efficiency. When enabled, it requires each
            element in dataset be a dict with keys "width" and "height".
        num_workers (int): number of parallel data loading workers

    Returns:
        torch.utils.data.DataLoader:
            a dataloader. Each output from it is a ``list[mapped_element]`` of length
            ``total_batch_size / num_workers``, where ``mapped_element`` is produced
            by the ``mapper``.
    """
    if isinstance(dataset, list):
        dataset = DatasetFromList(dataset, copy=False)
    if mapper is not None:
        dataset = MapDataset(dataset, mapper)
    if sampler is None:
        sampler = TrainingSampler(len(dataset))
    assert isinstance(sampler, torch.utils.data.sampler.Sampler)
    return build_batch_data_loader(
        dataset,
        sampler,
        total_batch_size,
        aspect_ratio_grouping=aspect_ratio_grouping,
        num_workers=num_workers,
    )

def _test_loader_from_config(cfg, dataset_name, mapper=None):
    """
    Uses the given `dataset_name` argument (instead of the names in cfg), because the
    standard practice is to evaluate each test set individually (not combining them).
    """
    if isinstance(dataset_name, str):
        dataset_name = [dataset_name]

    dataset = get_detection_dataset_dicts(
        dataset_name,
        filter_empty=False,
        proposal_files=[
            cfg.DATASETS.PROPOSAL_FILES_TEST[list(
                cfg.DATASETS.TEST).index(dataset_name)]
        ]
        if cfg.MODEL.LOAD_PROPOSALS
        else None,
        cfg=cfg
    )
    if mapper is None:
        mapper = DatasetMapper(cfg, False)
    return {"dataset": dataset, "mapper": mapper, "num_workers": cfg.DATALOADER.NUM_WORKERS}

@configurable(from_config=_test_loader_from_config)
def build_detection_test_loader(dataset, *, mapper, sampler=None, num_workers=0):
    """
    Similar to `build_detection_train_loader`, but uses a batch size of 1,
    and :class:`InferenceSampler`. This sampler coordinates all workers to
    produce the exact set of all samples.
    This interface is experimental.

    Args:
        dataset (list or torch.utils.data.Dataset): a list of dataset dicts,
            or a map-style pytorch dataset. They can be obtained by using
            :func:`DatasetCatalog.get` or :func:`get_detection_dataset_dicts`.
        mapper (callable): a callable which takes a sample (dict) from dataset
           and returns the format to be consumed by the model.
           When using cfg, the default choice is ``DatasetMapper(cfg, is_train=False)``.
        sampler (torch.utils.data.sampler.Sampler or None): a sampler that produces
            indices to be applied on ``dataset``. Default to :class:`InferenceSampler`,
            which splits the dataset across all workers.
        num_workers (int): number of parallel data loading workers

    Returns:
        DataLoader: a torch DataLoader, that loads the given detection
        dataset, with test-time transformation and batching.

    Examples:
    ::
        data_loader = build_detection_test_loader(
            DatasetRegistry.get("my_test"),
            mapper=DatasetMapper(...))

        # or, instantiate with a CfgNode:
        data_loader = build_detection_test_loader(cfg, "my_test")
    """
    if isinstance(dataset, list):
        dataset = DatasetFromList(dataset, copy=False)
    if mapper is not None:
        dataset = MapDataset(dataset, mapper)
    if sampler is None:
        sampler = InferenceSampler(len(dataset))
    # Always use 1 image per worker during inference since this is the
    # standard when reporting inference time in papers.
    batch_sampler = torch.utils.data.sampler.BatchSampler(
        sampler, 1, drop_last=False)
    data_loader = torch.utils.data.DataLoader(
        dataset,
        num_workers=num_workers,
        batch_sampler=batch_sampler,
        collate_fn=trivial_batch_collator,
    )
    return data_loader


import asyncio
import logging
from typing import Iterator, Union

import grpc
from django.conf import settings
from pydantic import Field

from llmstack.common.blocks.base.schema import BaseSchema
from llmstack.common.runner.proto import runner_pb2, runner_pb2_grpc
from llmstack.connections.models import (
    Connection,
    ConnectionActivationInput,
    ConnectionActivationOutput,
    ConnectionStatus,
    ConnectionType,
)
from llmstack.connections.types import ConnectionTypeInterface

logger = logging.getLogger(__name__)

class WebLoginBaseConfiguration(BaseSchema):
    _storage_state: str = Field(
        description='Storage state', widget='textarea', hidden=True)

class WebLoginConfiguration(WebLoginBaseConfiguration):
    start_url: str = Field(description='URL to login to',
                           default='https://google.com')

class WebLogin(ConnectionTypeInterface[WebLoginConfiguration]):
    @staticmethod
    def name() -> str:
        return 'Web Login'

    @staticmethod
    def provider_slug() -> str:
        return 'promptly'

    @staticmethod
    def slug() -> str:
        return 'web_login'

    @staticmethod
    def description() -> str:
        return 'Login to a website'

    @staticmethod
    def type() -> ConnectionType:
        return ConnectionType.BROWSER_LOGIN

    def input(self, activation_input: ConnectionActivationInput) -> None:
        if activation_input.data == 'terminate':
            self._is_terminated = True

    async def _request_iterator(self, connection, timeout):
        session_data = connection.configuration['_storage_state'] if '_storage_state' in connection.configuration and connection.configuration['_storage_state'] else ''
        yield runner_pb2.RemoteBrowserRequest(init_data=runner_pb2.BrowserInitData(url=connection.configuration['start_url'],
                                                                                terminate_url_pattern='',
                                                                                session_data=session_data,
                                                                                timeout=timeout, persist_session=True))

        # Sleep till timeout or self._is_terminated is True
        time_elapsed = 0
        while not self._is_terminated and time_elapsed < timeout:
            await asyncio.sleep(0.1)
            time_elapsed += 0.1

        yield runner_pb2.RemoteBrowserRequest(input=runner_pb2.BrowserInput(type=runner_pb2.TERMINATE))

    async def _read_browser_output(self, connection, remote_browser_stream):
        while not self._is_terminated:
            try:
                # Read the next message from the stream
                output = await remote_browser_stream.read()
                if output.state is not runner_pb2.RemoteBrowserState.RUNNING:
                    self._is_terminated = True
                    if output.session.session_data:
                        connection.configuration['_storage_state'] = output.session.session_data
                        connection.status = ConnectionStatus.ACTIVE
                    else:
                        connection.status = ConnectionStatus.FAILED
                    break
            except grpc.aio.AioRpcError as e:
                # Handle the case where the stream has been closed by the server
                if e.code() == grpc.StatusCode.CANCELLED:
                    # Stream has been cancelled, likely no more messages to read
                    logger.info('Stream was cancelled, no more messages.')
                    break
                else:
                    # An actual error occurred, log or re-raise
                    logger.error(
                        f'RPC error occurred: {e.code()} - {e.details()}')
                    raise
            except asyncio.CancelledError:
                # The task was cancelled, so exit the loop
                logger.info('Task was cancelled, exiting read loop.')
                break
            except TimeoutError:
                pass

            await asyncio.sleep(0.01)

        return connection

    async def activate(self, connection) -> Iterator[Union[Connection, dict]]:
        self.channel = grpc.aio.insecure_channel(
            f'{settings.RUNNER_HOST}:{settings.RUNNER_PORT}')
        stub = runner_pb2_grpc.RunnerStub(self.channel)
        self._input_index = 0
        self._input_instructions = []
        self._is_terminated = False
        timeout = 120

        self.remote_browser_stream = stub.GetRemoteBrowser(
            self._request_iterator(connection, timeout))

        first_message = await self.remote_browser_stream.read()

        ws_url = first_message.session.ws_url

        yield ConnectionActivationOutput(data={'ws_url': ws_url, 'timeout': timeout})

        # Consume the stream in a separate task and return the connection
        connection = await self._read_browser_output(connection, self.remote_browser_stream)

        await self.channel.close()
        yield connection