import logging
import json
import os
from common import utils
from table_order.table_order_item_list import TableOrderItemList
# 環境変数の取得
LOGGER_LEVEL = os.environ.get("LOGGER_LEVEL")
# ログ出力の設定
logger = logging.getLogger()
if LOGGER_LEVEL == 'DEBUG':
    logger.setLevel(logging.DEBUG)
else:
    logger.setLevel(logging.INFO)

# ログ出力の設定
logger = logging.getLogger()
logger.setLevel(logging.INFO)

# テーブル操作クラスの初期化
item_master_table_controller = TableOrderItemList()

def get_category():
    """
    カテゴリIDとカテゴリ名を返却する

    Returns
    -------
    categories:dict
        商品カテゴリ一覧情報
    """

    categories = item_master_table_controller.scan()
    for category in categories:
        category.pop('items')
    return categories

def lambda_handler(event, context):
    """
    商品カテゴリ情報を返す
    Parameters
    ----------
    event : dict
        フロントより渡されたパラメータ
    context : dict
        コンテキスト内容。
    Returns
    -------
    categories : dict
        商品カテゴリ一覧情報
    """
    # パラメータログ
    logger.info(event)
    try:
        categories = get_category()
    except Exception as e:
        logger.error('Occur Exception: %s', e)
        return utils.create_error_response('ERROR')

    return utils.create_success_response(
        json.dumps(categories, default=utils.decimal_to_int,
                   ensure_ascii=False))


from gluonts.dataset.util import to_pandas
import logging
import pandas as pd
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from gluonts.dataset.field_names import FieldName
import os
import numpy as np
from pandas.tseries import offsets
from pandas.tseries.frequencies import to_offset
TARGET_DIM = 4
def time_format_from_frequency_str(freq_str: str) :
    """
    get suitable time_stamp format for this frequency
    Parameters
    ----------

    freq_str
        Frequency string of the form [multiple][granularity] such as "12H", "5min", "1D" etc.
        B       business day frequency
        C       custom business day frequency (experimental)
        D       calendar day frequency
        W       weekly frequency
        M       month end frequency
        BM      business month end frequency
        CBM     custom business month end frequency
        MS      month start frequency
        BMS     business month start frequency
        CBMS    custom business month start frequency
        Q       quarter end frequency
        BQ      business quarter endfrequency
        QS      quarter start frequency
        BQS     business quarter start frequency
        A       year end frequency
        BA      business year end frequency
        AS      year start frequency
        BAS     business year start frequency
        BH      business hour frequency
        H       hourly frequency
        T, min  minutely frequency
        S       secondly frequency
        L, ms   milliseonds
        U, us   microseconds
        N       nanoseconds

    """

    features_by_offsets = {
        offsets.YearOffset: '%Y',
        offsets.MonthOffset: '%Y-%m',
        offsets.Week: '%Y-%W',
        offsets.Day: '%Y-%m-%d',
        offsets.BusinessDay: '%Y-%m-%d',
        offsets.Hour: '%Y-%m-%d %H:%M:%S',
        offsets.Minute: '%Y-%m-%d %H:%M:%S',
    }

    offset = to_offset(freq_str)

    for offset_type, format_pattern in features_by_offsets.items():
        if isinstance(offset, offset_type):
            return format_pattern

def create_dataset_if_not_exist(paths, start, past_length , pred_length , slice , timestep , freq):
    for name in paths:
        path = paths[name]
        if not os.path.exists(path):
            logging.info('there is no dataset [%s] , creating...' % name)
            os.system('python data_process/preprocessing.py -st="{}" -d="{}" -t="{}" -p="{}" -s="{}" -n="{}" -f="{}"'
                      .format( start
                              , name
                              , past_length
                              , pred_length
                              , slice
                              , timestep
                              , freq))
        else:
            logging.info(' dataset [%s] was found , good~~~' % name)

def add_time_mark_to_file(path):
    '''
    add time for the duplicated files
    '''
    count = 1
    if not os.path.exists(path):
        return path
    file_name_list = os.path.splitext(os.path.basename(path))
    father = os.path.split(path)[0]

    new_path = os.path.join(father ,file_name_list[0]+'_%d'%(count)+file_name_list[1])
    while os.path.exists(new_path):
        count += 1
        new_path = os.path.join(father, file_name_list[0] + '_%d'%(count) + file_name_list[1])

    return new_path

def add_time_mark_to_dir(path):
    '''
    add time for the duplicated directory
    '''
    if not os.path.exists(path):
        return path
    count = 1;
    father = os.path.split(path)[0]
    dir_name = os.path.split(path)[-1]

    new_path = os.path.join(father ,dir_name+'_%d'%(count))
    while os.path.exists(new_path):
        count += 1
        new_path = os.path.join(father ,dir_name+'_%d'%(count))

    return new_path

def weighted_average(
   metrics, weights = None, axis=None
):
    """
    compute weighted average
    metrics  #(ssm_num , bs,seq_length )
    weights  #(ssm_num , bs , seq_length)
    axis
    """
    import tensorflow as tf
    if weights is not None:
        weighted_tensor = metrics * weights
        sum_weights = tf.math.maximum(1.0, tf.math.reduce_sum(weights,axis=axis))
        return tf.math.reduce_sum(weighted_tensor,axis=axis) / sum_weights
    else:
        return tf.math.reduce_mean(metrics, axis=axis)

def sample_without_put_back(samples, size):

    result = []
    for i in range(size):
        z = np.random.choice(samples, 1)[0]
        result.append(z)
        index = np.where(samples == z)
        samples = np.delete(samples, index)
    return result

def plot_train_pred(path, targets_name, data, pred, batch, epoch, plot_num, plot_length, time_start, freq, nan_data=0):
    '''
    :param path: path to store picture
    :param targets_name: dataset name
    :param data: real dataset #(ssm_num , bs , seq , 1)
    :param nan_data: fill missing data with nan_data
    :param pred: prediction length
    :param batch: batch number
    :param epoch: epoch number
    :param plot_num: picture number(random choice)
    :param plot_length: training length
    :param time_start : start time of the picture
    :param freq : frequency of the dataset
    :return:
    '''
    data = np.squeeze(data , -1)
    pred = np.squeeze(pred ,-1)
    root_path = os.path.join(path , 'train_pred_pic')
    if plot_num > data.shape[1]:
        plot_num = data.shape[1]
    samples_no = sample_without_put_back(np.arange(data.shape[1]) , plot_num)
    current_dir = os.path.join(root_path, 'epoch({})'.format(epoch))
    if not os.path.isdir(current_dir):
        os.makedirs(current_dir)
    for ssm_no in np.arange(data.shape[0]):
        for sample in samples_no:
            pic_name = os.path.join(current_dir, 'batch_no({})_dataset({})_sample({})'.format(batch, targets_name.split(',')[ssm_no], sample))
            time_range = pd.date_range(time_start[sample], periods=data.shape[2], freq=freq)
            time_range = time_range[-plot_length:]
            s1 = data[ssm_no , sample ,-plot_length:]
            s1[s1 == nan_data] = np.NaN
            s2 = pred[ssm_no , sample ,-plot_length:]
            fig = plt.figure(figsize=(28, 21))

            ax = fig.add_subplot(1, 1, 1)
            ax.set_title('DATASET({}) EPOCHE({} prediction result)'.format(targets_name.split(',')[ssm_no], epoch), fontsize=30)
            ax.plot(time_range, s1, linestyle='-', color='tab:green', marker='D', label='Truth')
            ax.plot(time_range, s2, linestyle='-.', color='tab:blue', marker='o', label='pred_mean')
            ax.xaxis.set_tick_params(labelsize=21)
            ax.yaxis.set_tick_params(labelsize=21)
            ax.legend(prop={'size': 31}, edgecolor='red', facecolor='#e8dfdf')
            plt.savefig(pic_name)
            plt.close(fig)

def plot_train_pred_NoSSMnum(path, targets_name, data, pred, batch, epoch, ssm_no, plot_num, plot_length, time_start, freq, nan_data=0):
    '''
    the same as `plot_train_pred`, but without dimension SSM
    '''
    # decline extra dimension
    data = np.squeeze(data , -1) #(bs, seq)
    pred = np.squeeze(pred ,-1)
    root_path = os.path.join(path , 'train_pred_pic')
    if plot_num > data.shape[1]:
        plot_num = data.shape[1]
    samples_no = sample_without_put_back(np.arange(data.shape[0]) , plot_num)
    current_dir = os.path.join(root_path, 'epoch({})'.format(epoch))
    if not os.path.isdir(current_dir):
        os.makedirs(current_dir)
    for sample in samples_no:
        pic_name = os.path.join(current_dir , 'batch_no({})_ssm_no({})_sample({})'.format(batch,ssm_no,sample))
        time_range = pd.date_range(time_start[sample], periods=data.shape[1], freq=freq)
        time_range = time_range[-plot_length:]
        s1 = data[sample, -plot_length:]
        s1[s1 == nan_data] = np.NaN
        s2 = pred[sample , -plot_length:]

        fig = plt.figure(figsize = (28,21))
        ax = fig.add_subplot(1, 1, 1)
        ax.set_title('DATASET({}) EPOCHE({} prediction result)'.format(targets_name.split(',')[ssm_no], epoch), fontsize=30)
        ax.plot(time_range , s1 , linestyle = '-' , color = 'tab:green' , marker = 'D' ,label = 'Truth')
        ax.plot(time_range , s2 , linestyle = '-.' , color = 'tab:blue' , marker = 'o' , label = 'prediction')
        ax.xaxis.set_tick_params(labelsize=21)
        ax.yaxis.set_tick_params(labelsize=21)
        ax.legend(prop={'size': 31},edgecolor='red', facecolor='#e8dfdf')
        plt.savefig(pic_name)
        plt.close(fig)
    pass

def plot_train_epoch_loss(
        result:dict,
        path: str,
        time: str
):
    epoches = list(result.keys())
    loss = list(result.values())
    fig = plt.figure(figsize=(16,12))
    ax = fig.add_subplot(1, 1, 1)
    ax.plot(epoches, loss, color='tab:blue' , marker='D')

    font = {
        'family': 'Times New Roman',
         'weight': 'normal',
         'size': 30,
    }
    title_font ={
        'family': 'Times New Roman',
        'weight': 'bold',
        'size': 40,
    }
    ax.set_xlabel('epoch(s)' ,font )
    ax.set_ylabel('loss(MSE / filter negative log likelihood)' , font)
    ax.set_title('epoch information when training' ,title_font )
    ax.xaxis.set_tick_params(labelsize=21)
    ax.yaxis.set_tick_params(labelsize=21)
    plt.savefig(os.path.join(path , 'train_%s.pdf'%(time)),format="pdf")
    plt.close(fig)

def complete_batch(batch,batch_size):
    """
        complete the batch withoug enough samples
    """
    # deepcopy
    completed_batch = batch.copy()
    if len(completed_batch[FieldName.START]) == batch_size:
        return completed_batch , batch_size
    for attr in completed_batch.keys():
        batch_value = completed_batch[attr]
        if isinstance(batch_value , list):
            batch_num = len(batch_value)
            completed_batch[attr] = batch_value + [batch_value[-1]]*(batch_size-batch_num)
        elif isinstance(batch_value, np.ndarray):
            if len(batch_value.shape) != TARGET_DIM:
                batch_num = batch_value.shape[0]
                complete_shape = [batch_size-batch_num]+list(batch_value.shape[1:])
                completed_batch[attr] = np.concatenate([batch_value , np.zeros(shape=complete_shape)], axis=0)
            else:
                batch_num = batch_value.shape[1]
                complete_shape = [batch_value.shape[0],batch_size - batch_num] + list(batch_value.shape[2:])
                completed_batch[attr] = np.concatenate([batch_value, np.zeros(shape=complete_shape)], axis=1)
    return completed_batch ,batch_num

def del_previous_model_params(path):
    '''
    delete previous model parameters
    '''
    if not os.path.isdir(path):
        return
    for file in os.listdir(path):
        if file.endswith(".data-00000-of-00001") \
                or file.endswith(".index") \
                or file.endswith(".meta"):
            os.remove(os.path.join(path,file))

def get_model_params_name(path):
    '''
    get params name
    '''
    if not os.path.isdir(path):
        return
    for file in os.listdir(path):
        if file.endswith(".data-00000-of-00001") \
                or file.endswith(".index") \
                or file.endswith(".meta"):
            params_name = os.path.splitext(file)[0]
            return params_name

def samples_with_mean_cov(mean: np.ndarray , cov:np.ndarray , num_samples: int):
    '''
    :param mean: (ssm_num, bs, pred, dim_z)
    :param cov: (ssm_num, bs, pred, dim_z ， dim_z)
    :return: samples (ssm_num ,bs,num_samples, pred, dim_z)
    '''
    result = np.zeros(shape=mean.shape)
    result =  np.tile(np.expand_dims(result,0),[100]+[1]*len(result.shape))#(samples, ssm ,bs, pred, dim_z)

    for i in range(mean.shape[0]):
        for j in range(mean.shape[1]):
            for k in range(mean.shape[2]):
                samples = np.random.multivariate_normal(mean[i,j,k] , cov[i,j,k] , size=num_samples)
                result[: , i, j , k] = samples
                # print(samples.shape)
    result = np.transpose(result ,[1,2,0,3,4])
    return result
    pass

def get_reload_hyper(path, config):
    '''
    reload config
    '''
    abbre = {
        'freq' : 'freq',
        'env' : 'environment',
        'lags' : 'maxlags',
        'past' : 'past_length',
        'pred' : 'pred_length',
        'u' : 'dim_u',
        'l' : 'dim_l',
        'K' : 'K',
        'T' : ['time_exact_layers' , 'time_exact_cells'],
        'E' : ['env_exact_layers' , 'env_exact_cells'],
        'α' : 'alpha_units',
        'epoch' : 'epochs',
        'bs' : 'batch_size',
        'bn' : 'num_batches_per_epoch',
        'lr' : 'learning_rate' ,
        'initKF' : 'init_kf_matrices',
        'dropout':'dropout_rate'
    }

    not_convert_to_int = ['freq' ,'env' , 'T' , 'E']
    convert_to_float = ['lr' , 'initKF' , 'dropout']
    hyper_parameters = path.split('_');
    for parameter in hyper_parameters:
        index_of_left_embrace = parameter.index('(')
        index_of_right_embrace = parameter.index(')')
        name = parameter[:index_of_left_embrace]
        value = parameter[index_of_left_embrace+1 : index_of_right_embrace]
        if name in convert_to_float :
            value = float(value)
        elif name not in not_convert_to_int:
            value = int(value)
        else:
            #维持原样就好了
            pass

        #单参数
        if isinstance(abbre[name] , str):
            config.__setattr__(abbre[name] , value)
        #多参数，形容LSTM
        else:
            value_split = value.split('-')
            for i in range(len(abbre[name])):
                config.__setattr__(abbre[name][i] , int(value_split[i]))

    return config


"""Parse the YAML configuration."""
import logging
from pathlib import Path
from typing import Optional, Dict, Callable, Tuple, Union

import yaml
import torch

from . import utils

logger = logging.getLogger("casanovo")

class Config:
    """The Casanovo configuration options.

    If a parameter is missing from a user's configuration file, the default
    value is assumed.

    Parameters
    ----------
    config_file : str, optional
        The provided user configuration file.

    Examples
    --------
    ```
    config = Config("casanovo.yaml")
    config.n_peaks    # the n_peaks parameter
    config["n_peaks"] # also the n_peaks parameter
    ```
    """

    _default_config = Path(__file__).parent / "config.yaml"
    _config_types = dict(
        random_seed=int,
        n_peaks=int,
        min_mz=float,
        max_mz=float,
        min_intensity=float,
        remove_precursor_tol=float,
        max_charge=int,
        precursor_mass_tol=float,
        isotope_error_range=lambda min_max: (int(min_max[0]), int(min_max[1])),
        min_peptide_len=int,
        dim_model=int,
        n_head=int,
        dim_feedforward=int,
        n_layers=int,
        dropout=float,
        dim_intensity=int,
        max_length=int,
        residues=dict,
        n_log=int,
        tb_summarywriter=str,
        warmup_iters=int,
        max_iters=int,
        learning_rate=float,
        weight_decay=float,
        train_batch_size=int,
        predict_batch_size=int,
        n_beams=int,
        top_match=int,
        max_epochs=int,
        num_sanity_val_steps=int,
        train_from_scratch=bool,
        save_model=bool,
        model_save_folder_path=str,
        save_weights_only=bool,
        every_n_train_steps=int,
        no_gpu=bool,
    )

    def __init__(self, config_file: Optional[str] = None):
        """Initialize a Config object."""
        self.file = str(config_file) if config_file is not None else "default"
        with self._default_config.open() as f_in:
            self._params = yaml.safe_load(f_in)

        if config_file is None:
            self._user_config = {}
        else:
            with Path(config_file).open() as f_in:
                self._user_config = yaml.safe_load(f_in)

        # Validate:
        for key, val in self._config_types.items():
            self.validate_param(key, val)

        # Add extra configuration options and scale by the number of GPUs.
        n_gpus = 0 if self["no_gpu"] else torch.cuda.device_count()
        self._params["n_workers"] = utils.n_workers()
        if n_gpus > 1:
            self._params["train_batch_size"] = (
                self["train_batch_size"] // n_gpus
            )

    def __getitem__(self, param: str) -> Union[int, bool, str, Tuple, Dict]:
        """Retrieve a parameter"""
        return self._params[param]

    def __getattr__(self, param: str) -> Union[int, bool, str, Tuple, Dict]:
        """Retrieve a parameter"""
        return self._params[param]

    def validate_param(self, param: str, param_type: Callable):
        """Verify a parameter is the correct type.

        Parameters
        ----------
        param : str
            The Casanovo parameter
        param_type : Callable
            The expected callable type of the parameter.
        """
        try:
            param_val = self._user_config.get(param, self._params[param])
            if param == "residues":
                residues = {
                    str(aa): float(mass) for aa, mass in param_val.items()
                }
                self._params["residues"] = residues
            elif param_val is not None:
                self._params[param] = param_type(param_val)
        except (TypeError, ValueError) as err:
            logger.error(
                "Incorrect type for configuration value %s: %s", param, err
            )
            raise TypeError(
                f"Incorrect type for configuration value {param}: {err}"
            )

    def items(self) -> Tuple[str, ...]:
        """Return the parameters"""
        return self._params.items()


from scipy import stats
import numpy as np
import csv

sentiment_scores = np.random.random(52)

# input should be a numpy array of sentiment scores for the 50 states
def  calculateRegressions(sentiment_scores):

	white_ratios = []
	male_ratios = []
	educations = []
	populations = []
	incomes = []

	criteria = ("white ratio", "male ratio", "education level", "population", "median household income")
	r_values = []
	extents_of_correlation = []
	demographics = [("white ratio", white_ratios, 1),
					("male ratio", male_ratios, 2),
					("education level", educations, 6),
					("population", populations, 7),
					("median household income", incomes, 8)]

	with open('USdemographics_by_state.csv', 'r') as file:
		reader = csv.reader(file)
		# skip header
		next(reader)

		for row in reader:
			for (_, data, col) in demographics:
				num = row[col].replace(",", "", 10)
				num = num.replace("$", "")
				data.append(float(num))

	correlation = [[0.3, "not"],
					[0.5, "weakly"],
					[0.7, "moderately"],
					[1.0, "strongly"]]

	for (colname, data, _) in demographics:
		slope, intercept, r_value, p_value, std_err = stats.linregress(sentiment_scores, data)
		# print for debugging purposes
		# print(colname + " r value: " + str(r_value))
		r_values.append(str(r_value))

		abs_r = abs(r_value)
		for (num, extent) in correlation:
			if abs_r < num:
				# print for debugging purposes
				# print(extent + " correlated")
				extents_of_correlation.append(extent + " correlated")
				break

	file.close()
	# output is a tuple of 3 tuples of strings
	return (criteria, tuple(r_values), tuple(extents_of_correlation))

print(calculateRegressions(sentiment_scores))


"""To run this: $ python3 parse.py test.xml
The script will pase a XML file and print its node tags.
Compatible with Python 3; changing the print statements should make this
compatible with Python 2.
"""

import sys

# http://docs.python.org/library/xml.etree.elementtree.html
from xml.etree import ElementTree

def operation(node):
    """Just a sample function that prints the tag of a node."""
    return print(node.tag)

def recur_node(node, f):
    """Applies function f on given node and goes down recursively to its
       children.

       Keyword arguments:
       node - the root node
       f - function to be applied on node and its children

    """
    if node != None:
        f(node)
        for item in node.getchildren():
            recur_node(item, f)
    else:
        return 0

def main(fileName):
    try:
        root = ElementTree.parse(fileName).getroot()
    except:
        # there should be some proper exception handling here
        return -1
    return recur_node(root, operation)

if __name__ == "__main__":
  sys.exit(main(sys.argv[1]))

###
###


import pygame as pg
from numpy import zeros

from constants import consts as c
from id_mapping import id_map, reverse_id_map

class World:
    def __init__(self):
        self.grid = zeros((c.num_cells, c.num_cells), dtype=int)
        self.ore_locations = []

        self.grid[10:13, 10:12] = id_map["coal"]
        self.grid[15:17, 15:17] = id_map["iron_ore"]
        self.grid[15:17, 20:23] = id_map["copper_ore"]
        self.populate_ore_locations()

    def populate_ore_locations(self):
        for row in range(c.num_cells):
            for col in range(c.num_cells):
                if self.grid[row, col] > 0:
                    self.ore_locations.append((row, col))

    def render(self):
        for loc in self.ore_locations:
            row, col = loc
            x = col * c.cell_length - c.player_x
            y = row * c.cell_length - c.player_y
            pg.draw.rect(c.screen, c.ore_colors[self.grid[loc]], (x, y, c.cell_length, c.cell_length))

    def render_tooltip(self, row, col):
        x, y = pg.mouse.get_pos()
        ore = reverse_id_map[self.grid[row, col]].replace("_", " ").title()
        ore_text = c.merriweather.render(ore, True, pg.Color("white"))
        pg.draw.rect(c.screen, pg.Color("black"), (x + 10, y + 10, ore_text.get_width() + 20, ore_text.get_height() + 20))
        c.screen.blit(ore_text, (x + 20, y + 20))

world = World()


import copy
import logging
import traceback
from typing import Any, Dict, Optional, List

from .base_trigger import TriggerEvent, BaseTrigger
from .playbook_utils import merge_global_params
from .playbooks_event_handler import PlaybooksEventHandler
from ..model.events import ExecutionBaseEvent
from ..reporting.base import Finding
from ...model.playbook_action import PlaybookAction
from ...model.config import Registry
from .trigger import Trigger

class PlaybooksEventHandlerImpl(PlaybooksEventHandler):
    def __init__(self, registry: Registry):
        self.registry = registry

    def handle_trigger(self, trigger_event: TriggerEvent) -> Optional[Dict[str, Any]]:
        playbooks = self.registry.get_playbooks().get_playbooks(trigger_event)
        if not playbooks:  # no registered playbooks for this event type
            return

        execution_response = None
        execution_event: Optional[ExecutionBaseEvent] = None
        findings: Dict[str, Finding] = {}
        for playbook in playbooks:
            fired_trigger = self.__get_fired_trigger(trigger_event, playbook.triggers)
            if fired_trigger:
                execution_event = fired_trigger.build_execution_event(
                    trigger_event, findings
                )
                if execution_event:  # might not exist for unsupported k8s types
                    execution_event.named_sinks = (
                        playbook.sinks
                        if playbook.sinks is not None
                        else self.registry.get_playbooks().get_default_sinks()
                    )

                    playbook_resp = self.__run_playbook_actions(
                        execution_event,
                        playbook.get_actions(),
                    )
                    if (
                        playbook_resp
                    ):  # For now, only last response applies. (For simplicity reasons)
                        execution_response = playbook_resp
                    if playbook.stop or execution_event.stop_processing:
                        break

        if execution_event:
            self.__handle_findings(execution_event)

        return execution_response

    def run_actions(
        self,
        execution_event: ExecutionBaseEvent,
        actions: List[PlaybookAction],
    ) -> Optional[Dict[str, Any]]:
        if execution_event.named_sinks is None:
            execution_event.named_sinks = (
                self.registry.get_playbooks().get_default_sinks()
            )

        execution_response = self.__run_playbook_actions(
            execution_event,
            actions,
        )
        self.__handle_findings(execution_event)

        return execution_response

    def __prepare_execution_event(self, execution_event: ExecutionBaseEvent):
        execution_event.set_scheduler(self.registry.get_scheduler())

    def run_external_action(
        self,
        action_name: str,
        action_params: Optional[dict],
        sinks: Optional[List[str]],
    ) -> Optional[Dict[str, Any]]:
        action_def = self.registry.get_actions().get_action(action_name)
        if not action_def:
            return self.__error_resp(f"External action not found {action_name}")

        if not action_def.from_params_func:
            return self.__error_resp(
                f"Action {action_name} cannot run using external event"
            )

        if sinks:
            if action_params:
                action_params["named_sinks"] = sinks
            else:
                action_params = {"named_sinks": sinks}
        try:
            instantiation_params = action_def.from_params_parameter_class(
                **action_params
            )
        except Exception:
            return self.__error_resp(
                f"Failed to create execution instance for"
                f" {action_name} {action_def.from_params_parameter_class}"
                f" {action_params} {traceback.format_exc()}"
            )

        execution_event = action_def.from_params_func(instantiation_params)
        if not execution_event:
            return self.__error_resp(
                f"Failed to create execution event for "
                f"{action_name} {action_params}"
            )

        playbook_action = PlaybookAction(
            action_name=action_name, action_params=action_params
        )
        return self.run_actions(execution_event, [playbook_action])

    @classmethod
    def __error_resp(cls, msg: str) -> dict:
        logging.error(msg)
        return {"success": False, "msg": msg}

    def __run_playbook_actions(
        self,
        execution_event: ExecutionBaseEvent,
        actions: List[PlaybookAction],
    ) -> Dict[str, Any]:
        self.__prepare_execution_event(execution_event)
        execution_event.response = {"success": True}
        for action in actions:
            if execution_event.stop_processing:
                return execution_event.response

            registered_action = self.registry.get_actions().get_action(
                action.action_name
            )

            if (
                not registered_action
            ):  # Might happen if manually trying to trigger incorrect action
                msg = f"action {action.action_name} not found. Skipping for event {type(execution_event)}"
                execution_event.response = self.__error_resp(msg)
                continue

            if not isinstance(execution_event, registered_action.event_type):
                msg = f"Action {action.action_name} requires {registered_action.event_type}"
                execution_event.response = self.__error_resp(msg)
                continue

            if not registered_action.params_type:
                registered_action.func(execution_event)
            else:
                action_params = None
                try:
                    action_params = merge_global_params(
                        self.get_global_config(), action.action_params
                    )
                    params = registered_action.params_type(**action_params)
                except Exception:
                    msg = (
                        f"Failed to create {registered_action.params_type} "
                        f"using {action_params} for running {action.action_name} "
                        f"exc={traceback.format_exc()}"
                    )
                    execution_event.response = self.__error_resp(msg)
                    continue

                registered_action.func(execution_event, params)

        return execution_event.response

    @classmethod
    def __get_fired_trigger(
        cls, trigger_event: TriggerEvent, playbook_triggers: List[Trigger]
    ) -> Optional[BaseTrigger]:
        for trigger in playbook_triggers:
            if trigger.get().should_fire(trigger_event):
                return trigger.get()
        return None

    def __handle_findings(self, execution_event: ExecutionBaseEvent):
        for finding in execution_event.findings.values():
            for sink_name in execution_event.named_sinks:
                try:
                    sink = self.registry.get_sinks().sinks.get(sink_name)
                    if not sink:
                        logging.error(
                            f"sink {sink_name} not found. Skipping event finding {finding}"
                        )
                        continue
                    # create deep copy, so that iterating on one sink won't affect the others
                    finding_copy = copy.deepcopy(finding)
                    sink.write_finding(finding_copy, self.registry.get_sinks().platform_enabled)
                except Exception:  # Failure to send to one sink shouldn't fail all
                    logging.error(
                        f"Failed to publish finding to sink {sink_name}", exc_info=True
                    )

    def get_global_config(self) -> dict:
        return self.registry.get_playbooks().get_global_config()


"""Analyze word-level LIWC scores between lucid and non-lucid dreams.
***Note this does not re-run LIWC, but loads in prior results.

IMPORTS
=======
    - original post info,     dreamviews-posts.tsv
    - word-level LIWC scores, validate-liwc_wordscores.tsv
    - LIWC dictionary,        dictionaries/custom.dic
EXPORTS
=======
    - effect sizes (d) for top words from each category, validate-liwc_wordscores-stats.tsv

Some of this is copy/pasted from the general liwc_stats script,
but this is way messier so better alone.
1. only a subset
2. need to load in numpy stuff
3. need to look up vocabs in whole lexicon for subsetting
4. don't need all the summary statistic details
5. don't want to run both stats on this, too out of hand just pick one
6. export is different, here it's the top N words of a few categories and their effect sizes
"""
import liwc
import numpy as np
import pandas as pd
import pingouin as pg
from scipy import sparse
import tqdm

import config as c

################################################################################
# SETUP
################################################################################

LIWC_CATEGORIES = ["insight", "agency"]
top_n = 20  # Top n contributing tokens/words for each category.

import_path_dict = c.DATA_DIR / "dictionaries" / "custom.dic"
import_path_data = c.DATA_DIR / "derivatives" / "validate-liwc_wordscores-data.npz"
import_path_attr = c.DATA_DIR / "derivatives" / "validate-liwc_wordscores-attr.npz"
export_path = c.DATA_DIR / "derivatives" / "validate-liwc_wordscores-stats.tsv"

#### load in the original posts file to get attributes lucidity and user_id
# and drop un-labeled posts.
# merge the clean data file and all its attributes with the liwc results
posts = c.load_dreamviews_posts()
posts = posts.set_index("post_id")[["user_id", "lucidity"]]
posts = posts[ posts["lucidity"].str.contains("lucid") ]

# #### load prior full LIWC results (i.e., category results)
# liwccats = pd.read_csv(import_fname_liwc, sep="\t", encoding="utf-8",
#     index_col="category", usecols=["category", "cohen-d"], squeeze=True)
# liwccats = liwccats.loc[LIWC_CATEGORIES]

#### load in dictionary lexicon
# and flip key/value from token/category to category/word_list
# and only for the liwc categories of interest
lexicon, _ = liwc.read_dic(import_fname_dict)
wordlists = { c: [ t for t, cats in lexicon.items() if c in cats ]
    for c in LIWC_CATEGORIES }
# and grab a subset of all the vocab so to reduce unnecessary memory later
vocab = set([ t for wlist in wordlists.values() for t in wlist ])

#### load in numpy arrays of token scores and generate dataframe
sparse_matr_attributes = np.load(import_fname_attr, allow_pickle=True)
tokens = sparse_matr_attributes["token"]
token_index = [ t in vocab for t in tokens ]
relevant_tokens = tokens[token_index]
sparse_matr = sparse.load_npz(import_fname_data)
scores = pd.DataFrame(sparse_matr[:,token_index].toarray(),
    columns=relevant_tokens,
    index=sparse_matr_attributes["post_id"])

# merge the post attributes with LIWC token scores
df = posts.join(scores, how="left")
assert len(df) == len(posts)

################################################################################
# STATISTICAL TESTS
################################################################################

# Average the LD and NLD scores of each token for each user.
# Some users might not have both dream types and they'll be removed.
avgs = df.groupby(["user_id", "lucidity"]
    ).mean().rename_axis(columns="token"
    ).pivot_table(index="user_id", columns="lucidity"
    ).dropna()

# We already have only relevant tokens, so get effect
# sizes for all of them.
effectsize_results = []
for tok in tqdm.tqdm(relevant_tokens, desc="stats on word-level LIWC scores"):
    ld, nld = avgs[tok][["lucid", "nonlucid"]].T.values
    stats = {}
    stats["cohen-d"] = pg.compute_effsize(ld, nld, paired=True, eftype="cohen")
    stats["cohen-d_lo"], stats["cohen-d_hi"] = pg.compute_bootci(ld, nld,
        paired=True, func="cohen", method="cper",
        confidence=.95, n_boot=2000, decimals=4)
    effectsize_results.append(pd.DataFrame(stats, index=[tok]))

es_df = pd.concat(effectsize_results).rename_axis("token")

# Each LIWC category will utilize a different set of tokens/words.
# Find the top N contributors to the overall effect.
# Find the direction of overall effect by looking at previous LIWC category output.
token_rank_results = []
for cat in LIWC_CATEGORIES:
    # generate an index of relevant tokens for this category
    cat_index = es_df.index.map(lambda t: t in wordlists[cat])
    # extract only rows in this category
    df_ = es_df.loc[cat_index]
    # # get direction of effect and remove rows not in line
    # d_sign = np.sign(liwccats.loc[cat])
    # if d_sign > 0:
    #     df_ = df_[ df_["cohen-d"] > 0 ]
    # else:
    #     df_ = df_[ df_["cohen-d"] < 0 ]
    # sort by effect size
    df_ = df_.sort_values("cohen-d", ascending=False, key=abs)
    # take top rows and clean up a bit
    df_ = df_[:TOP_N]
    df_[f"{cat}_rank"] = np.arange(TOP_N) + 1
    token_rank_results.append(df_)

out = pd.concat(token_rank_results)

# Export.
out.to_csv(export_path, float_format="%.4f", index=True, na_rep="NA", sep="\t", encoding="utf-8")


#!/usr/bin/python
# -*- encoding: utf-8 -*-

#Created by 'quanpower' on '15-4-4'
#Email:quanpower@gmail.com
#QQ:252527676
#Site:www.smartlinkcloud.com

__author__ = 'quanpower'

import numpy as np
import cv2

#获得视频的格式
videoCapture =cv2.VideoCapture('../files/video_test.rmvb')

#获得码率及尺寸
fps = videoCapture.get(cv2.cv.CV_CAP_PROP_FPS)
size = (int(videoCapture.get(cv2.cv.CV_CAP_PROP_FRAME_WIDTH)),
        int(videoCapture.get(cv2.cv.CV_CAP_PROP_FRAME_HEIGHT)))

#指定写视频的格式, I420-avi, MJPG-mp4
videoWriter = cv2.VideoWriter('oto_other.mp4', cv2.cv.CV_FOURCC('M', 'J', 'P', 'G'), fps, size)

#读帧
success, frame = videoCapture.read()

while success :
    cv2.imshow('Oto Video', frame) #显示
    cv2.waitKey(1000/int(fps)) #延迟
    videoWriter.write(frame) #写视频帧
    success, frame = videoCapture.read() #获取下一帧

    if cv2.waitKey(1)&0xFF == ord('q'):
        break

videoCapture.release()
cv2.destroyAllWindows()


import datetime
import logging

import numpy as np
import os
import skimage.io as io
from sklearn.cluster import KMeans
from skimage import img_as_float
from skimage.color import rgb2lab
from skimage.exposure import equalize_adapthist
from skimage.filters import gaussian
from skimage.transform import resize

from src.data_loader import sample_names, images, root_dir, FOLDER_EXPERIMENTS
from src.utils import apply_on_normalized_luminance, colormap, outline_regions, average_color

MAX_PATIENTS = 1
MAX_IMAGES_PER_PATIENT = 1
MAX_PATCHES_PER_IMAGE = 2
RESIZE_IMAGES = None    # (300, 300)  # None to deactivate

if __name__ == "__main__":
    execution_id = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
    results_dir = root_dir(FOLDER_EXPERIMENTS(version=3), execution_id)
    os.makedirs(results_dir, exist_ok=True)
    logging.basicConfig(
        level=logging.INFO,
        handlers=[
            logging.FileHandler(os.path.join(results_dir, 'log.txt')),
            logging.StreamHandler()
        ]
    )

    p_names = sample_names()
    for idx_p, p_name in enumerate(p_names[0:MAX_PATIENTS]):

        for idx_img, (path_image, image) in enumerate(images(patient_name=p_name, max_images=MAX_IMAGES_PER_PATIENT)):
            results_p_dir = os.path.join(results_dir, p_name, str(idx_img))
            os.makedirs(results_p_dir, exist_ok=True)
            logging.info(f'Processing: {p_name}-{idx_img}')

            # 01 Preprocessing
            ###

            if RESIZE_IMAGES:
                logging.info('Resizing image')
                image = resize(img_as_float(image), RESIZE_IMAGES + (3, ))
            io.imsave(fname=os.path.join(results_p_dir, '01 01 Original.jpg'),
                      arr=image)

            logging.info('Gaussian filter')
            image = apply_on_normalized_luminance(
                operation=gaussian,
                image_rgb=image)
            io.imsave(fname=os.path.join(results_p_dir, '01 02 Gaussian filter.jpg'),
                      arr=image)

            logging.info('CLAHE')
            image = apply_on_normalized_luminance(
                lambda img: equalize_adapthist(img, clip_limit=0.02),
                image_rgb=image)
            io.imsave(fname=os.path.join(results_p_dir, '01 03 CLAHE.jpg'),
                      arr=image)

            # 02 Lab visualization
            ###

            image_lab = rgb2lab(image)
            luminance = image_lab[:, :, 0]
            a = np.min(luminance)
            b = np.max(luminance - a)
            luminance = (luminance - a) / b
            io.imsave(fname=os.path.join(results_p_dir, '02 01 Luminance.png'),
                      arr=luminance)
            channel_a = image_lab[:, :, 1]
            a = np.min(channel_a)
            b = np.max(channel_a - a)
            channel_a = (channel_a - a) / b
            io.imsave(fname=os.path.join(results_p_dir, '02 01 channel_a.png'),
                      arr=channel_a)
            channel_b = image_lab[:, :, 2]
            a = np.min(channel_b)
            b = np.max(channel_b - a)
            channel_b = (channel_b - a) / b
            io.imsave(fname=os.path.join(results_p_dir, '02 01 channel_b.png'),
                      arr=channel_b)

            # 03-04 Lab separation
            ###

            logging.info('Positive separation (k-means clustering on `a channel`)')
            channel_a = image_lab[:, :, 1]
            clustering = KMeans(n_clusters=2, random_state=0).fit(channel_a.reshape(-1, 1))
            idx_positive_cluster = np.argmax(clustering.cluster_centers_)
            positive_mask = np.equal(clustering.labels_, idx_positive_cluster).reshape(channel_a.shape[0:2])
            io.imsave(fname=os.path.join(results_p_dir, f'03 K-means - Positives - Mask.jpg'),
                      arr=colormap(positive_mask))
            io.imsave(fname=os.path.join(results_p_dir, f'03 K-means - Positives - regions.jpg'),
                      arr=outline_regions(image=image, region_labels=positive_mask))
            io.imsave(fname=os.path.join(results_p_dir, f'03 K-means - Positives - average color.jpg'),
                      arr=average_color(image=image, region_labels=positive_mask))

            logging.info('Negative separation (k-means clustering on `b channel`)')
            channel_b = image_lab[:, :, 2]
            nonpositive_mask = np.logical_not(positive_mask)
            clustering = KMeans(n_clusters=2, random_state=0).fit(channel_b[nonpositive_mask].reshape(-1, 1))
            idx_negative_cluster = np.argmin(clustering.cluster_centers_)
            negative_mask_wrt_nonpositive = np.equal(clustering.labels_, idx_negative_cluster)
            negative_mask = np.full(shape=positive_mask.shape, dtype=bool, fill_value=False)
            nonpositive_mask_as_flatten_idcs = np.where(nonpositive_mask.flatten())
            negative_mask_wrt_nonpositive_flatten = nonpositive_mask_as_flatten_idcs[0][negative_mask_wrt_nonpositive]
            negative_mask = negative_mask.flatten()
            negative_mask[negative_mask_wrt_nonpositive_flatten] = True
            negative_mask = negative_mask.reshape(positive_mask.shape)
            io.imsave(fname=os.path.join(results_p_dir, f'04 K-means - Negatives - Mask.jpg'),
                      arr=colormap(negative_mask))
            io.imsave(fname=os.path.join(results_p_dir, f'04 K-means - Negatives - regions.jpg'),
                      arr=outline_regions(image=image, region_labels=negative_mask))
            io.imsave(fname=os.path.join(results_p_dir, f'04 K-means - Negatives - average color.jpg'),
                      arr=average_color(image=image, region_labels=negative_mask))

            # 05 Results visualization
            ###

            region_labels = negative_mask + 2*positive_mask
            io.imsave(fname=os.path.join(results_p_dir, '05 Results - labels.jpg'),
                      arr=colormap(region_labels))
            io.imsave(fname=os.path.join(results_p_dir, '05 Results - regions.jpg'),
                      arr=outline_regions(image=image, region_labels=region_labels))
            io.imsave(fname=os.path.join(results_p_dir, '05 Results - average_color.jpg'),
                      arr=average_color(image=image, region_labels=region_labels))


# -*- encoding: utf-8 -*-

import os
import time
import pyautogui as auto
import PyHook3 as pyhook
import pythoncom
from PIL import Image
from pytesseract import image_to_string
from googletrans import Translator
import asyncio
import tkinter as tk
# 引入字体模块
import tkinter.font as tkFont
from concurrent.futures import ThreadPoolExecutor

translator = Translator(service_urls=['translate.google.cn'])

px=-1000
py=-1000
x=-1000
y=-1000
last=0

root = tk.Tk("Advise")
root.overrideredirect(True)
sw=root.winfo_screenwidth()
sh=root.winfo_screenheight()
root_x=sw-200
root_y=sh-120
root.geometry("200x50+%d+%d"%(root_x,root_y))
root.wm_attributes('-topmost',1)
root.wm_attributes('-alpha',0.15)
font =  tkFont.Font(family="Microsoft YaHei",size=9);
english = tk.Label(root,text="English",font=font)
english.pack()
chinese = tk.Label(root,text="Chinese",font=font)
chinese.pack()

async def translateZh(text):
    global translator,english
    text=text.replace("\n"," ")
    english['text']=translator.translate(text,src="zh-cn",dest="en").text
    await asyncio.sleep(0.5)

async def translateEn(text):
    global translator
    text=text.replace("\n"," ")
    chinese['text']=translator.translate(text,src="en",dest="zh-cn").text
    await asyncio.sleep(0.5)

async def translate(text):
    global root
    task1 = asyncio.create_task(translateEn(text))
    task2 = asyncio.create_task(translateZh(text))
    await task1
    await task2
    root.wm_attributes('-alpha',0.4)
    root.update()
    await hide(5)

async def hide(delay):
    global root
    await asyncio.sleep(delay)
    root.wm_attributes('-alpha',0)
    root.update()

def getClickPosition(_x,_y):
    global x,y,px,py
    px=x;py=y;
    x=_x;y=_y;
    l=min(x,px)
    t=min(y,py)
    w=abs(px-x)
    h=abs(py-y)
    if w<1000 and h<1000:
        auto.screenshot("temp.png",region=(l,t,w,h))
        img=Image.open("temp.png")
        img=img.convert("L")
        text = image_to_string(img,"chi_sim+eng")
        if text:
            asyncio.run(translate(text))
        x=-1000
        y=-1000

def onMouseEvent(event):
    try:
        global last,loop
        if event.Message==514:
            # Judge Double Click
            if event.Time-last<300:
                getClickPosition(event.Position[0],event.Position[1])
            last=event.Time
        return True
    except KeyboardInterrupt:
        exit(0)

def main():
    hm = pyhook.HookManager()
    hm.MouseAll = onMouseEvent
    hm.HookMouse()
    pythoncom.PumpMessages()

if __name__ == "__main__":
    main()


# Объедините функции из прошлых задач.
# Функцию угадайку задекорируйте:
# ○ декораторами для сохранения параметров,
# ○ декоратором контроля значений и
# ○ декоратором для многократного запуска.
# Выберите верный порядок декораторов.

from typing import Callable
from random import randint
import os
import json

def check_parametr(func: Callable):
    MIN_NUM = 1
    MAX_NUM = 100
    MIN_COUNT = 1
    MAX_COUNT =10
    def wrapper(num: int, count: int, *args, **kwargs):
        if num > MAX_NUM or num < MIN_NUM:
            num = randint(MIN_NUM, MAX_NUM)
        if count > MAX_COUNT or count < MIN_COUNT:
            count = randint(MIN_COUNT, MAX_COUNT)
        # print(num, count)
        result = func(num, count, *args, **kwargs)

        return result

    return wrapper

def logger(func: Callable):
    file_name = f'{func.__name__}.json'
    if os.path.exists(file_name):
        with open(file_name, 'r', encoding='utf-8') as f:
            data = json.load(f)
    else:
        data = []

    def wrapper(*args, **kwargs):
        json_dict = {'args': args, **kwargs}
        result = func(*args, **kwargs)
        json_dict['result'] = result
        data.append(json_dict)

        with open(file_name, 'w', encoding='utf-8') as f1:
            json.dump(data, f1)

        return result

    return wrapper

def count_f(num: int = 1):
    def deco(func: Callable):
        results = []
        def wrapper(*args, **kwargs):
            for i in range(num):
                results.append(func(*args, **kwargs))
            return results
        return wrapper
    return deco

@count_f(2)
@check_parametr
@logger
def gess_number(num: int, count: int) -> Callable[[], None]:

    for i in range(1, count + 1):
        num_input = int(input(f'Угадайте число от 1 до 100, у Вас {count + 1 - i} попыток: '))
        print(f'Попытка № {i}')
        if num_input == num:
            return 'Вы угадали!'
        elif num_input < num:
            print('Загаданное число больше')
        else:
            print('Загаданное число меньше')

    return 'Вы не угадали!'

if __name__ == '__main__':
    game = gess_number(25, 5)
    print(game)


import os
import re
import logging
import json
from datetime import datetime, timedelta
from pathlib import Path

import requests
import spotipy
from spotipy import SpotifyOAuth
from spotipy.oauth2 import SpotifyClientCredentials
from dagster_cron import SystemCronScheduler
from dagster import (
    RepositoryDefinition,
    ScheduleDefinition,
    pipeline,
    schedules,
    solid,
)

from env import FEEDLY_REFRESH_TOKEN, SPOTIPY_CLIENT_ID, SPOTIPY_CLIENT_SECRET, SPOTIPY_REDIRECT_URI

class FeedlyApiRequestError(Exception):
    pass

class MusicFeeder:
    scopes = " ".join([
        "ugc-image-upload",
        "user-read-playback-state",
        "user-modify-playback-state",
        "user-read-currently-playing",
        "streaming",
        "app-remote-control",
        "user-read-email",
        "user-read-private",
        "playlist-read-collaborative",
        "playlist-modify-public",
        "playlist-read-private",
        "playlist-modify-private",
        "user-library-modify",
        "user-library-read",
        "user-top-read",
        "user-read-recently-played",
        "user-follow-read",
        "user-follow-modify"
    ])
    feedly_username = '5066bdab-2d04-4094-b7ac-cd19047daffe'
    spotify_username = "21bpmv7tcs4av5sakwfxkof2a"
    spotify_playlist_id = "6qsodHkxMB36VhWfIwvylQ"
    stop_words = ["MV"]

    def __init__(self):
        # feedly setup
        self.feedly_access_token = self._get_feedly_access_token()
        # spotify setup
        self.cache_path = Path(__file__).parent / f".cache-{self.spotify_username}"
        self.sp_oauth = SpotifyOAuth(
            SPOTIPY_CLIENT_ID,
            SPOTIPY_CLIENT_SECRET,
            SPOTIPY_REDIRECT_URI,
            scope=self.scopes,
            cache_path=self.cache_path,
            username=self.spotify_username
        )
        self.token_info = self.sp_oauth.get_cached_token()
        self.token = self.token_info["access_token"]
        self.spotify = spotipy.Spotify(auth=self.token)
        self.titles, self.tracks, self.current_tracks, self.tracks_to_add = None, None, None, None

    def execute(self):
        self.titles = self._get_titles()
        # extract latest 30 titles
        self.titles = self.titles[:30]
        self.tracks = self._get_taget_tracks(self.titles)
        self.current_tracks = self._get_current_tracks()
        self.tracks_to_add = self._get_tracks_to_add(self.tracks, self.current_tracks)
        self._add_to_spotify(self.tracks_to_add)

    def _get_feedly_access_token(self):
        url = "https://cloud.feedly.com/v3/auth/token"
        payload = dict(
            refresh_token=FEEDLY_REFRESH_TOKEN,
            client_id="feedlydev",
            client_secret="feedlydev",
            grant_type="refresh_token"
        )
        resp = requests.post(url, json=payload).json()
        return resp['access_token']

    def _request_to_feedly(self, url, params={}):
        headers = {'Authorization': f"OAuth {self.feedly_access_token}"}
        resp = requests.get(url, headers=headers, params=params)
        if resp.status_code != 200:
            raise FeedlyApiRequestError(json.dumps(resp.json()))

        return resp.json()

    def _get_titles(self):
        url = f"https://cloud.feedly.com/v3/streams/contents?streamId=user/{self.feedly_username}/category/Music"
        params = dict(unreadOnly=False, count=100)
        resp = self._request_to_feedly(url, params)
        return [item['title'] for item in resp['items'] if item['title']]

    def _get_taget_tracks(self, titles):
        one_week_ago = datetime.today() - timedelta(days=7)
        tracks = []
        for title in titles:
            terms = re.findall("[a-zA-Z ]+", title)
            terms = [term.strip() for term in terms if term.strip()]
            terms = [term for term in terms if term not in self.stop_words]
            query = " ".join(terms)
            logging.info(f"searching: {query}")
            result = self.spotify.search(query)['tracks']['items']
            if result:
                track = result[0]
                try:
                    release_date = datetime.strptime(track['album']['release_date'], "%Y-%m-%d")
                except ValueError:
                    continue

                # Don't append if the release date of the track is older than one week ago
                if release_date < one_week_ago:
                    continue
                tracks.append(track['id'])

        return tracks

    def _get_current_tracks(self):
        return self.spotify.user_playlist(self.spotify_username, self.spotify_playlist_id)['tracks']['items']

    def _get_tracks_to_add(self, tracks, current_tracks):
        current_track_ids = [c_track['track']['id'] for c_track in current_tracks]
        tracks = [track for track in tracks if track not in current_track_ids]
        return tracks

    def _add_to_spotify(self, tracks):
        if not tracks:
            return
        self.spotify.user_playlist_add_tracks(self.spotify_username, self.spotify_playlist_id, tracks, position=0)

@solid
def load_to_spotify(context):
    MusicFeeder().execute()

@pipeline
def load_to_spotify_pipeline():
    load_to_spotify()

def etl_spotify_repository():
    return RepositoryDefinition(
        'etl_spotify_repository', pipeline_defs=[load_to_spotify_pipeline]
    )

@schedules(SystemCronScheduler)
def spotify_schedules():
    return [
        ScheduleDefinition(
            name='daily_spotify_batch',
            cron_schedule='0 * * * *',
            pipeline_name='load_to_spotify_pipeline',
            environment_dict={},
        )
    ]

if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO)
    MusicFeeder().execute()
    # EtlFeedlyspotify(context=context).execute()
    # result = execute_pipeline(load_to_spotify_pipeline)
    # assert result.success

# def test_hello_cereal_solid():
#     res = execute_solid(hello_cereal)
#     assert res.success
#     assert len(res.output_value()) == 77

# def test_hello_cereal_pipeline():
#     res = execute_pipeline(hello_cereal_pipeline)
#     assert res.success
#     assert len(res.result_for_solid('hello_cereal').output_value()) == 77


from splinter import Browser
from bs4 import BeautifulSoup
import time
import pandas as pd
import pymongo
import requests

def init_browser():
    executable_path = {"executable_path": "chromedriver.exe"}
    return Browser("chrome", **executable_path, headless=False)

def scrape():
    browser = init_browser()

    # NASA Mars News

    url= "https://mars.nasa.gov/news/"
    browser.visit(url)
    html= browser.html
    soup= BeautifulSoup(html,"html.parser")

    latest_news_title= soup.find("div", class_="bottom_gradient").find("h3").text
    p_text= soup.find("div", class_= "article_teaser_body").text

    # JPL Mars Space Images

    img_url= "https://www.jpl.nasa.gov/spaceimages/?search=&category=Mars"
    browser.visit(img_url)
    time.sleep(5)
    img_html= browser.html
    i_soup= BeautifulSoup(img_html,"html.parser")
    img_path= i_soup.find("article", class_= "carousel_item")\
    ["style"].replace("background-image: url(","").replace(");", "")[1:-1]
    mars_url= "https://www.jpl.nasa.gov"
    featured_image_url= mars_url+img_path

    # Mars Weather

    w_url= "https://twitter.com/marswxreport?lang=en"
    browser.visit(w_url)
    time.sleep(5)
    html= browser.html
    soup= BeautifulSoup(html, "html.parser")
    mars_weather = (soup.find('div', attrs={"data-testid": "tweet"}).get_text()).split('InSight ')[1]

    # Mars Facts

    f_url= "https://space-facts.com/mars/"
    browser.visit(f_url)
    time.sleep(5)
    m_facts= pd.read_html(f_url)
    mars_df= m_facts[0]
    mars_facts= mars_df.rename(columns={0: "Aspect", 1: "Detail"}).set_index("Aspect")
    mars_df_table= mars_facts.to_html()
    mars_df_table.replace("\n","")

    # Mars Hemispheres

    h_url= "https://astrogeology.usgs.gov/search/results?q=hemisphere+enhanced&k1=target&v1=Mars"
    browser.visit(h_url)
    time.sleep(5)
    h_html= browser.html
    soup= BeautifulSoup(h_html, "html.parser")

    h_images= soup.find("div", class_= "collapsible results")
    hems= h_images.find_all("a")

    hems_list= []

    for hemisphere in hems:
        if hemisphere.h3:
            title=hemisphere.h3.text
            link=hemisphere["href"]
            home_url="https://astrogeology.usgs.gov/"
            next_url=home_url+link
            browser.visit(next_url)
            html = browser.html
            soup = BeautifulSoup(html, 'html.parser')
            hemisphere2=soup.find("div",class_= "downloads")
            img=hemisphere2.ul.a["href"]
            hemisphere_dict={}
            hemisphere_dict["Title"]=title
            hemisphere_dict["Image_URL"]=img
            hems_list.append(hemisphere_dict)
            browser.back()

    hems_list

    Mars_info={
        "Mars_news_title": latest_news_title,
        "Mars_news_text": p_text,
        "Mars_featured_img": featured_image_url,
        "Mars_weather": mars_weather,
        "Mars_facts": mars_df_table,
        "Mars_hemispheres": hems_list
    }

    browser.quit()

    return Mars_info

# if __name__ == "__main__":
#     data = scrape()
#     print(data)


from torch.utils.data import Dataset
from torchvision import transforms
import numpy as np
import pandas as pd
import random
from src.dataset import get_load_data
import torch
from tqdm import tqdm

def generate_train_set(train):
    train_set = []
    classes = [i[1] for i in train]
    classes = np.array(classes)
    for i in tqdm(range(len(train))):

        # positive_list = [j for j in range(len(classes)) if classes[j] == train[i][1]]
        positive_list = np.argwhere(classes == train[i][1])
        positive_ind = list(random.choice(positive_list))[0]
        while positive_ind == i:
            positive_ind = list(random.choice(positive_list))[0]
        negative_list = np.argwhere(classes != train[i][1])
        negative_ind = list(random.choice(negative_list))[0]
        train_set.append({"anchor": train[i][0], "positive": train[positive_ind][0], "negative": train[negative_ind][0], "anchor_label": train[i][1]})

    return train_set

class TripletLossDataset(Dataset):
    def __init__(self, train_set = None, train=True, root = "data", dataset = "FashionMNIST"):

        self.is_train = train

        if self.is_train:
            if train_set is None:
                train, _ = get_load_data(root = root, dataset = dataset)
                self.train_set = generate_train_set(train)
            else:
                self.train_set = train_set
        else:
            _, self.test = get_load_data(root = root, dataset = dataset)

    def __len__(self):
        if self.is_train:
            return len(self.train_set)
        else:
            return len(self.test)

    def __getitem__(self, item):

        if self.is_train:
            data = self.train_set[item]

            return data['anchor'], data['positive'], data['negative'], data['anchor_label']

        else:
            return self.test[item][0], self.test[item][1]


from data_handler import DataHandler
from scanner import Scanner
from datetime import datetime
from queue import Queue
from threading import Thread
import sys

def printStats(devices_count, sensor_type):
    time = datetime.now().strftime("%H:%M:%S")
    print("%s | %d {:>4} devices found.".format(
        sensor_type) % (time, devices_count))

def queueScans(scanner):
    jobs = Queue()
    threads_list = list()

    t1 = Thread(target=lambda q: q.put(
        scanner.count_bt_devices()), args=(jobs,))
    t1.daemon = True  # Kill thread on exit
    t1.start()
    threads_list.append(t1)

    t2 = Thread(target=lambda q: q.put(
        scanner.count_wifi_devices()), args=(jobs,))
    t2.daemon = True  # Kill thread on exit
    t2.start()
    threads_list.append(t2)

    # Join threads
    for t in threads_list:
        t.join()
        # t.keepRunning = False

    scan_results = []

    # Get return values from threads
    while not jobs.empty():
        result = jobs.get()
        scan_results.append(result)
    return scan_results

def main():
    scanner = Scanner()
    data_handler = DataHandler()

    while True:
        try:
            data = queueScans(scanner)
            bt_devices_count = data[0]
            wifi_devices_count = data[1]

            sensor_type = "bt"
            printStats(bt_devices_count, sensor_type)
            # data_handler.send_data(bt_devices_count, sensor_type)
            data_handler.save_data(bt_devices_count, sensor_type)

            sensor_type = "wifi"
            printStats(wifi_devices_count, sensor_type)
            # data_handler.send_data(wifi_devices_count, sensor_type)
            data_handler.save_data(wifi_devices_count, sensor_type)

        except (KeyboardInterrupt, SystemExit):
            sys.exit()

if __name__ == "__main__":
    main()


# %%
from typing import Dict, List

from kfp import dsl

@dsl.component(base_image="zhuwq0/quakeflow:latest")
def cut_templates(
    root_path: str,
    region: str,
    config: Dict,
    index: int = 0,
    model_path: str = "../PhaseNet/",
    mseed_list: List = None,
    protocol: str = "file",
    bucket: str = "",
    token: Dict = None,
) -> str:
    import json
    import multiprocessing as mp
    import os
    from dataclasses import dataclass
    from datetime import datetime, timedelta, timezone
    from glob import glob
    from multiprocessing.pool import ThreadPool
    from pathlib import Path

    import gamma
    import numpy as np
    import obspy
    import pandas as pd
    from tqdm import tqdm

    def extract_template_numpy(
        template_fname,
        traveltime_fname,
        traveltime_index_fname,
        traveltime_type_fname,
        arrivaltime_index_fname,
        snr_fname,
        mseed_path,
        events,
        stations,
        picks,
        config,
        lock,
        ibar,
    ):
        template_array = np.memmap(template_fname, dtype=np.float32, mode="r+", shape=tuple(config["template_shape"]))
        traveltime_array = np.memmap(
            traveltime_fname, dtype=np.float32, mode="r+", shape=tuple(config["traveltime_shape"])
        )
        traveltime_index_array = np.memmap(
            traveltime_index_fname, dtype=np.int32, mode="r+", shape=tuple(config["traveltime_shape"])
        )
        traveltime_type_array = np.memmap(
            traveltime_type_fname, dtype=np.int32, mode="r+", shape=tuple(config["traveltime_shape"])
        )
        arrivaltime_index_array = np.memmap(
            arrivaltime_index_fname, dtype=np.int64, mode="r+", shape=tuple(config["traveltime_shape"])
        )
        snr_array = np.memmap(snr_fname, dtype=np.float32, mode="r+", shape=tuple(config["snr_shape"]))

        # %%
        tmp = mseed_path.split("/")
        year_jday, hour = tmp[-2], tmp[-1]
        begin_time = datetime.strptime(f"{year_jday}T{hour}", "%Y-%jT%H").replace(tzinfo=timezone.utc)
        end_time = begin_time + timedelta(hours=1)
        events_ = events[(events["event_time"] > begin_time) & (events["event_time"] < end_time)]

        if len(events_) == 0:
            return 0

        # %%
        waveforms_dict = {}
        for station_id in tqdm(
            stations["station_id"], desc=f"Loading waveform: ", position=ibar % 6, nrows=7, mininterval=5, leave=True
        ):
            for c in config["components"]:
                if os.path.exists(f"{mseed_path}/{station_id}{c}.mseed"):
                    try:
                        stream = obspy.read(f"{mseed_path}/{station_id}{c}.mseed")
                        stream.merge(fill_value="latest")
                        if len(stream) > 1:
                            print(f"More than one trace: {stream}")
                        trace = stream[0]
                        if trace.stats.sampling_rate != config["sampling_rate"]:
                            if trace.stats.sampling_rate % config["sampling_rate"] == 0:
                                trace.decimate(int(trace.stats.sampling_rate / config["sampling_rate"]))
                            else:
                                trace.resample(config["sampling_rate"])
                        # trace.detrend("linear")
                        # trace.taper(max_percentage=0.05, type="cosine")
                        # trace.filter("bandpass", freqmin=1.0, freqmax=15.0, corners=2, zerophase=True)
                        waveforms_dict[f"{station_id}{c}"] = trace
                    except Exception as e:
                        print(e)
                        continue

        # %%
        picks["station_component_index"] = picks.apply(lambda x: f"{x.station_id}.{x.phase_type}", axis=1)

        # %%
        num_event = 0
        for ii, event_index in tqdm(
            enumerate(events_["event_index"]),
            total=len(events_),
            desc=f"Cutting event {year_jday}T{hour}",
            position=ibar % 6,
            nrows=7,
            mininterval=5,
            leave=True,
        ):
            if event_index not in picks.index:
                continue

            picks_ = picks.loc[[event_index]]
            picks_ = picks_.set_index("station_component_index")

            event_loc = events_.loc[event_index][["x_km", "y_km", "z_km"]].to_numpy().astype(np.float32)
            event_loc = np.hstack((event_loc, [0]))[np.newaxis, :]
            station_loc = stations[["x_km", "y_km", "z_km"]].to_numpy()

            template_ = np.zeros((6, len(stations), config["nt"]), dtype=np.float32)
            snr_ = np.zeros((6, len(stations)), dtype=np.float32)
            traveltime_ = np.zeros((2, len(stations)), dtype=np.float32)
            traveltime_type_ = np.zeros((2, len(stations)), dtype=np.int32)
            arrivaltime_index_ = np.zeros((2, len(stations)), dtype=np.int64)

            for i, phase_type in enumerate(["P", "S"]):
                traveltime = gamma.seismic_ops.calc_time(
                    event_loc,
                    station_loc,
                    [phase_type.lower() for _ in range(len(station_loc))],
                    vel={"p": 6.0, "s": 6.0 / 1.73},
                ).squeeze()

                phase_timestamp_pred = events_.loc[event_index]["event_timestamp"] + traveltime
                # predicted_phase_time = [events_.loc[event_index]["event_time"] + pd.Timedelta(seconds=x) for x in traveltime]

                mean_shift = []
                for j, station_id in enumerate(stations["station_id"]):
                    if f"{station_id}.{phase_type}" in picks_.index:
                        ## TODO: check if multiple phases for the same station
                        phase_timestamp = picks_.loc[f"{station_id}.{phase_type}"]["phase_timestamp"]
                        phase_timestamp_pred[j] = phase_timestamp
                        mean_shift.append(
                            phase_timestamp - (events_.loc[event_index]["event_timestamp"] + traveltime[j])
                        )

                        traveltime[j] = phase_timestamp - events_.loc[event_index]["event_timestamp"]
                        traveltime_type_[i, j] = 1
                        arrivaltime_index_[i, j] = int(round(phase_timestamp * config["sampling_rate"]))
                        # arrivaltime_index_[i, j] = phase_timestamp
                    else:
                        traveltime_type_[i, j] = 0

                # if len(mean_shift) > 0:
                #     mean_shift = float(np.median(mean_shift))
                # else:
                #     mean_shift = 0
                # phase_timestamp_pred[traveltime_type_[i, :] == 0] += mean_shift
                # traveltime[traveltime_type_[i, :] == 0] += mean_shift
                traveltime_[i, :] = traveltime

                for j, station in enumerate(stations.iloc):
                    station_id = station["station_id"]

                    empty_data = True
                    for c in station["component"]:
                        c_index = i * 3 + config["component_mapping"][c]

                        if f"{station_id}{c}" in waveforms_dict:
                            trace = waveforms_dict[f"{station_id}{c}"]

                            begin_time = (
                                phase_timestamp_pred[j]
                                - trace.stats.starttime.datetime.replace(tzinfo=timezone.utc).timestamp()
                                - config["time_before"]
                            )
                            end_time = (
                                phase_timestamp_pred[j]
                                - trace.stats.starttime.datetime.replace(tzinfo=timezone.utc).timestamp()
                                + config["time_after"]
                            )

                            trace_data = trace.data[
                                max(0, int(begin_time * trace.stats.sampling_rate)) : max(
                                    0, int(end_time * trace.stats.sampling_rate)
                                )
                            ].astype(np.float32)

                            if len(trace_data) < config["nt"]:
                                continue
                            std = np.std(trace_data)
                            if std == 0:
                                continue

                            empty_data = False
                            template_[c_index, j, : config["nt"]] = trace_data[: config["nt"]]
                            s = np.std(trace_data[-int(config["time_after"] * config["sampling_rate"]) :])
                            n = np.std(trace_data[: int(config["time_before"] * config["sampling_rate"])])
                            if n == 0:
                                snr_[c_index, j] = 0
                            else:
                                snr_[c_index, j] = s / n

            # template_array[event_index] = template_
            # traveltime_array[event_index] = traveltime_
            # traveltime_index_array[event_index] = np.round(traveltime_ * config["sampling_rate"]).astype(np.int32)
            # traveltime_type_array[event_index] = traveltime_type_
            # arrivaltime_index_array[event_index] = arrivaltime_index_
            # snr_array[event_index] = snr_
            template_array[ii] = template_
            traveltime_array[ii] = traveltime_
            traveltime_index_array[ii] = np.round(traveltime_ * config["sampling_rate"]).astype(np.int32)
            traveltime_type_array[ii] = traveltime_type_
            arrivaltime_index_array[ii] = arrivaltime_index_
            snr_array[ii] = snr_

            with lock:
                template_array.flush()
                traveltime_array.flush()
                traveltime_index_array.flush()
                traveltime_type_array.flush()
                arrivaltime_index_array.flush()
                snr_array.flush()

            # num_event += 1
            # if num_event > 20:
            #     break

    # %%
    result_path = f"{region}/cctorch"
    if not os.path.exists(f"{root_path}/{result_path}"):
        os.makedirs(f"{root_path}/{result_path}")

    # %%
    stations = pd.read_json(f"{root_path}/{region}/obspy/stations.json", orient="index")
    stations["station_id"] = stations.index
    # stations = stations[
    #     (stations["longitude"] >= config.xlim_degree[0])
    #     & (stations["longitude"] =< config.xlim_degree[1])
    #     & (stations["latitude"] >= config.ylim_degree[0])
    #     & (stations["latitude"] <= config.ylim_degree[1])
    # ]
    # stations["distance_km"] = stations.apply(
    #     lambda x: math.sqrt((x.latitude - config.latitude0) ** 2 + (x.longitude - config.longitude0) ** 2)
    #     * config.degree2km,
    #     axis=1,
    # )
    # stations.sort_values(by="distance_km", inplace=True)
    # stations.drop(columns=["distance_km"], inplace=True)
    # stations.sort_values(by="latitude", inplace=True)
    stations["x_km"] = stations.apply(
        lambda x: (x.longitude - config["longitude0"]) * np.cos(np.deg2rad(config["latitude0"])) * config["degree2km"],
        axis=1,
    )
    stations["y_km"] = stations.apply(lambda x: (x.latitude - config["latitude0"]) * config["degree2km"], axis=1)
    stations["z_km"] = stations.apply(lambda x: -x["elevation_m"] / 1e3, axis=1)

    # %%
    events = pd.read_csv(f"{root_path}/{region}/gamma/gamma_events.csv", parse_dates=["time"])
    events = events[events["time"].notna()]
    events.sort_values(by="time", inplace=True)
    events.rename(columns={"time": "event_time"}, inplace=True)
    events["event_time"] = events["event_time"].apply(lambda x: pd.to_datetime(x, utc=True))
    events["event_timestamp"] = events["event_time"].apply(lambda x: x.timestamp())
    events["x_km"] = events.apply(
        lambda x: (x.longitude - config["longitude0"]) * np.cos(np.deg2rad(config["latitude0"])) * config["degree2km"],
        axis=1,
    )
    events["y_km"] = events.apply(lambda x: (x.latitude - config["latitude0"]) * config["degree2km"], axis=1)
    events["z_km"] = events.apply(lambda x: x.depth_km, axis=1)

    # %%
    if "event_index" not in events.columns:
        event_index = events.index
    else:
        event_index = list(events["event_index"])
    event_index_fname = f"{root_path}/{result_path}/event_index.txt"
    with open(event_index_fname, "w") as f:
        for i, idx in enumerate(event_index):
            f.write(f"{i},{idx}\n")
    config["cctorch"]["event_index_file"] = event_index_fname

    # %%
    picks = pd.read_csv(
        f"{root_path}/{region}/gamma/gamma_picks.csv",
        parse_dates=["phase_time"],
    )
    picks = picks[picks["event_index"] != -1]
    picks["phase_timestamp"] = picks["phase_time"].apply(lambda x: x.timestamp())

    picks_ = picks.groupby("station_id").size()
    # station_id_ = picks_[picks_ > (picks_.sum() / len(picks_) * 0.1)].index
    # stations = stations[stations["station_id"].isin(station_id_)]
    stations = stations[stations["station_id"].isin(picks_.index)]

    stations.to_json(f"{root_path}/{result_path}/stations_filtered.json", orient="index", indent=4)
    stations.to_csv(f"{root_path}/{result_path}/stations_filtered.csv", index=True, index_label="station_id")

    station_index_fname = f"{root_path}/{result_path}/station_index.txt"
    with open(station_index_fname, "w") as f:
        for i, sta in enumerate(stations.iloc):
            f.write(f"{i},{sta['station_id']},{sta['component']}\n")
    config["cctorch"]["station_index_file"] = station_index_fname

    # %%
    picks = picks.merge(stations, on="station_id")
    picks = picks.merge(events, on="event_index", suffixes=("_station", "_event"))

    # %%
    events["index"] = events["event_index"]
    events.set_index("index", inplace=True)
    picks["index"] = picks["event_index"]
    picks.set_index("index", inplace=True)

    # %%
    nt = int((config["cctorch"]["time_before"] + config["cctorch"]["time_after"]) * config["cctorch"]["sampling_rate"])
    config["cctorch"]["nt"] = nt
    nch = 6  ## For [P,S] phases and [E,N,Z] components
    # nev = int(events.index.max()) + 1
    nev = len(events)
    nst = len(stations)
    print(f"nev: {nev}, nch: {nch}, nst: {nst}, nt: {nt}")
    template_shape = (nev, nch, nst, nt)
    traveltime_shape = (nev, nch // 3, nst)
    snr_shape = (nev, nch, nst)
    config["cctorch"]["template_shape"] = template_shape
    config["cctorch"]["traveltime_shape"] = traveltime_shape
    config["cctorch"]["snr_shape"] = snr_shape

    template_fname = f"{root_path}/{result_path}/template.dat"
    traveltime_fname = f"{root_path}/{result_path}/traveltime.dat"
    traveltime_index_fname = f"{root_path}/{result_path}/traveltime_index.dat"
    traveltime_type_fname = f"{root_path}/{result_path}/traveltime_type.dat"
    arrivaltime_index_fname = f"{root_path}/{result_path}/arrivaltime_index.dat"
    snr_fname = f"{root_path}/{result_path}/snr.dat"
    config["cctorch"]["template_file"] = template_fname
    config["cctorch"]["traveltime_file"] = traveltime_fname
    config["cctorch"]["traveltime_index_file"] = traveltime_index_fname
    config["cctorch"]["traveltime_type_file"] = traveltime_type_fname
    config["cctorch"]["arrivaltime_index_file"] = arrivaltime_index_fname
    config["cctorch"]["snr_file"] = snr_fname

    template_array = np.memmap(template_fname, dtype=np.float32, mode="w+", shape=template_shape)
    traveltime_array = np.memmap(traveltime_fname, dtype=np.float32, mode="w+", shape=traveltime_shape)
    traveltime_index_array = np.memmap(traveltime_index_fname, dtype=np.int32, mode="w+", shape=traveltime_shape)
    traveltime_type_array = np.memmap(traveltime_type_fname, dtype=np.int32, mode="w+", shape=traveltime_shape)
    arrivaltime_index_array = np.memmap(arrivaltime_index_fname, dtype=np.int64, mode="w+", shape=traveltime_shape)
    snr_array = np.memmap(snr_fname, dtype=np.float32, mode="w+", shape=snr_shape)

    with open(f"{root_path}/{result_path}/config.json", "w") as f:
        json.dump(config["cctorch"], f, indent=4, sort_keys=True)

    # %%
    dirs = sorted(glob(f"{root_path}/{region}/waveforms/????-???/??"))
    ncpu = mp.cpu_count()
    lock = mp.Lock()

    # with mp.get_context("spawn").Pool(ncpu) as pool:
    with ThreadPool(ncpu) as pool:
        pool.starmap(
            extract_template_numpy,
            [
                (
                    template_fname,
                    traveltime_fname,
                    traveltime_index_fname,
                    traveltime_type_fname,
                    arrivaltime_index_fname,
                    snr_fname,
                    d,
                    events,
                    stations,
                    picks,
                    config["cctorch"],
                    lock,
                    i,
                )
                for i, d in enumerate(dirs)
            ],
        )

if __name__ == "__main__":
    import json
    import os

    root_path = "local"
    region = "demo"
    with open(f"{root_path}/{region}/config.json", "r") as fp:
        config = json.load(fp)

    cut_templates.python_func(root_path, region=region, config=config)


import os
import re
import xlrd
import xlwt
from xlutils.copy import copy

class HandleExcel:
    def __init__(self):
        self.root = ''
        self.dir = ''
        self.files = ''
        self.allFile_url = ''
        self.final_file_name = '测试数据清洗版.xls'
        self.title = ['条码内容', '故障通道', '测试时间']
        self.file_name = '测试数据汇总.xls'

    def get_allfile_msg(self,file_dir):
        for root, dirs, files in os.walk(file_dir):
            return root, dirs, files

    def get_allfile_url(self,root, files):
        allFile_url = []
        for f in files:
            file_url = root + '\\' + f
            allFile_url.append(file_url)
        return allFile_url

    def all_to_one(self,root, allFile_url, files, file_name='allExcel.xls', title=None):
        file_name = root + '\\' + file_name
        self.create_excel(file_name, title)

        list_row_data = []
        for f in allFile_url:
            excel = xlrd.open_workbook(f)
            table = excel.sheet_by_index(0)
            for i, j in zip(range(table.nrows), files):
                if i == 0:
                    continue
                row = table.row_values(i)
                k = re.sub("\D", "", j)
                row.append(k)
                list_row_data.append(row)
        self.add_row(list_row_data, file_name)

    def create_excel(self,file_name, title):
        a = xlwt.Workbook()
        table = a.add_sheet('sheet1', cell_overwrite_ok=True)
        for i in range(len(title)):
            table.write(0, i, title[i])
        a.save(file_name)

    file_name = '测试数据汇总.xls'
    title = ['条码内容', '故障通道', '测试时间']

    def add_row(self,list_row_data, file_name):
        allExcel1 = xlrd.open_workbook(file_name)
        sheet = allExcel1.sheet_by_index(0)
        allExcel2 = copy(allExcel1)
        sheet2 = allExcel2.get_sheet(0)
        i = 0
        for row_data in list_row_data:
            for j in range(len(row_data)):
                sheet2.write(sheet.nrows + i, j, row_data[j])
            i += 1  # 覆盖原文件
        allExcel2.save(file_name)

    def clear_excel(self,file_dir,file_name):
        data = xlrd.open_workbook(file_dir + '\\' + file_name)
        table = data.sheets()[0]
        nrows = table.nrows
        ncols = table.ncols

        first_ncol = []
        second_ncol = []
        thrid_ncol = []

        for i in range(1, nrows):
            first_ncol_value = table.cell(i, 0).value
            first_ncol.append(first_ncol_value)

        for j in range(1, nrows):
            second_ncol_value = table.cell(j, 1).value
            second_ncol.append(second_ncol_value)

        for k in range(1, nrows):
            thrid_ncol_value = table.cell(k, 2).value
            thrid_ncol.append(thrid_ncol_value)

        clear_code_list = []
        clear_result_list = []
        clear_time_list = []

        for x, y, z in zip(first_ncol, second_ncol, thrid_ncol):
            if x not in clear_code_list and y == '\'PASS\'' or y == 'PASS' and x.startswith('A'):
                clear_code_list.append(x)
                clear_result_list.append(y)
                clear_time_list.append(z)

        a = xlwt.Workbook()
        table = a.add_sheet('sheet1', cell_overwrite_ok=True)
        # 写表头
        for i in range(len(self.title)):
            table.write(0, i, self.title[i])
        # 写内容
        for code_value, result_value, time_value in zip(range(len(clear_code_list)), range(len(clear_result_list)),
                                                        range(len(clear_time_list))):
            table.write(code_value + 1, 0, clear_code_list[code_value])
            table.write(result_value + 1, 1, clear_result_list[code_value])
            table.write(time_value + 1, 2, clear_time_list[code_value])

        a.save(file_dir + '\\' + self.final_file_name)


#!/usr/bin/env python3

import os
import logging
import argparse
import csv

def process_result(result_folder):
    status = "success"
    summary = []
    paths = []
    tests = ["TLPWhere", "TLPGroupBy", "TLPHaving", "TLPWhereGroupBy", "TLPDistinct", "TLPAggregate"]

    for test in tests:
        err_path = '{}/{}.err'.format(result_folder, test)
        out_path = '{}/{}.out'.format(result_folder, test)
        if not os.path.exists(err_path):
            logging.info("No output err on path %s", err_path)
            summary.append((test, "SKIPPED"))
        elif not os.path.exists(out_path):
            logging.info("No output log on path %s", out_path)
        else:
            paths.append(err_path)
            paths.append(out_path)
            with open(err_path, 'r') as f:
                if 'AssertionError' in f.read():
                    summary.append((test, "FAIL"))
                    status = 'failure'
                else:
                    summary.append((test, "OK"))

    logs_path = '{}/logs.tar.gz'.format(result_folder)
    if not os.path.exists(logs_path):
        logging.info("No logs tar on path %s", logs_path)
    else:
        paths.append(logs_path)
    stdout_path = '{}/stdout.log'.format(result_folder)
    if not os.path.exists(stdout_path):
        logging.info("No stdout log on path %s", stdout_path)
    else:
        paths.append(stdout_path)
    stderr_path = '{}/stderr.log'.format(result_folder)
    if not os.path.exists(stderr_path):
        logging.info("No stderr log on path %s", stderr_path)
    else:
        paths.append(stderr_path)

    description = "SQLancer test run. See report"

    return status, description, summary, paths

def write_results(results_file, status_file, results, status):
    with open(results_file, 'w') as f:
        out = csv.writer(f, delimiter='\t')
        out.writerows(results)
    with open(status_file, 'w') as f:
        out = csv.writer(f, delimiter='\t')
        out.writerow(status)

if __name__ == "__main__":
    logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')
    parser = argparse.ArgumentParser(description="ClickHouse script for parsing results of sqlancer test")
    parser.add_argument("--in-results-dir", default='/test_output/')
    parser.add_argument("--out-results-file", default='/test_output/test_results.tsv')
    parser.add_argument("--out-status-file", default='/test_output/check_status.tsv')
    args = parser.parse_args()

    state, description, test_results, logs = process_result(args.in_results_dir)
    logging.info("Result parsed")
    status = (state, description)
    write_results(args.out_results_file, args.out_status_file, test_results, status)
    logging.info("Result written")


import pathlib
import math
import pygame as pg
# bd is short for board
bd_size = 900
grid_size = bd_size // 19
stone_radius = grid_size // 2
cross_half_len = stone_radius // 3
center_circle = [(3, 3), (3, 9), (3, 15), (9, 3), (9, 9), (9, 15),
                    (15, 3), (15, 9), (15, 15)]
circle_radius = stone_radius // 5
base_dir = pathlib.Path(__file__).parent.parent
resource_dir = base_dir / 'resource'
bg_img_path = resource_dir / 'img' / 'back.png'
bg_music_path = resource_dir / 'music' / 'back.flac'
bg_music_volume = 0.6
bg_music_diff = 0.15
white_id = 0
black_id = 1

notice_width_grids = 8
notice_height_grids = 10
notice_width = grid_size * notice_width_grids
notice_height = grid_size * notice_height_grids
notice_leftup_coord = (5, 4)

header_font = pg.font.get_default_font()
header_size = grid_size
text_font = pg.font.get_default_font()
text_size = grid_size

class QuitException(Exception):
    pass


#!/usr/bin/python
import argparse
import time
import struct
import socket
import select
import sys

class PingOptions(object):

    def __init__(self):
        """Reset the class; indicates the class hasn't been initailized"""
        self.initialized = False

    def initialize(self, parser):
        parser.add_argument('--host', required=True, help='')
        parser.add_argument('--packet-size', type=int, default=32, help='')
        parser.add_argument('--ping-times', type=int, default=5, help='')
        return parser

    def gather_options(self):

        if not self.initialized:
            parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
            parser = self.initialize(parser)

        # opt, _ = parser.parse_known_args()
        return parser.parse_args()

    def print_options(self, opt):

        message = ''
        message += '----------------- Options ---------------\n'
        for k, v in sorted(vars(opt).items()):
            comment = ''
            default = self.parser.get_default(k)
            if v != default:
                comment = '\t[default: %s]' % str(default)
            message += '{:>25}: {:<30}{}\n'.format(str(k), str(v), comment)
        message += '----------------- End -------------------'
        print(message)

    def parse(self):

        opt = self.gather_options()
        self.opt = opt
        return self.opt


__author__ = 'vamsee'

import geojson as geo

Center = (-83.79706,42.268138)
Center_Point = geo.Point(Center)

def get_locations(filename,lat_idx,long_idx, write_filename,point_properties):
    opened_file = open(filename)
    features_list = []
    for line in opened_file:
        line_contents = line.split(',')
        (longitude,latitude) = (float(line_contents[lat_idx])/10000000,float(line_contents[long_idx])/10000000 )
        f = geo.Feature(geometry=geo.Point( (longitude,latitude)), \
            properties = point_properties )
        features_list.append(f)
        #break
    F = geo.FeatureCollection(features_list)
    # generate data
    geo_data_str = geo.dumps(F,sort_keys=True)

    # Write into file
    json_file  = open('../Mapbox/data/'+write_filename,'w')
    json_file.write(geo_data_str)
    json_file.close()

# get_locations('../../Transportation-Data/Data/RSE_BSM/RSE_BSM_trimmed.csv')
get_locations('../BSM-RSE.csv',7,6,'vehicle-locations.json',{'title':'Vehicle Location',  "marker-symbol": "car", "marker-size": "medium", "marker-color": "#09f"})
get_locations('../Geometry.csv',4,3,'rse-data-locations.json',{'title':'RSE Location',  "marker-symbol": "commercial", "marker-size": "large", "marker-color": "#f83"})


import pymc as pm
import numpy as np

x = np.random.normal(loc=0, scale=1, size=10) # Data Placeholder
y = np.random.normal(loc=2*x, scale=1, size=10) # Data Placeholder

with pm.Model() as model:

	# Priors
	α = pm.Normal("α", 1, 2)
	β = pm.Normal("β", -1, 2)
	σ = pm.Exponential("σ", 3)

	# Linear Model
	μ = pm.Deterministic("μ", α + β*x)

	# Likelihood
	y_ = pm.Normal("y", μ, σ, observed=y)

with model:
	trace = pm.sample()

if __name__ == "__main__":

	from pykrusch import krusch
	krusch(model, outname="../img/simple_model_posterior.png", posterior_trace=trace)


#!/usr/bin/env python

import math
import copy

import matplotlib.pyplot as plt

def plot_obstacles(previous_obstacles, new_obstacles, updated_obstacles, updated_labels):
    # Extract x and y coordinates from the obstacles lists
    prev_x, prev_y, new_x, new_y, updated_x, updated_y  = [], [],[],[],[],[]
    if len(previous_obstacles)>0:
        prev_x, prev_y = zip(*previous_obstacles)
    if len(new_obstacles)>0:
        new_x, new_y = zip(*new_obstacles)
    if len(updated_obstacles)>0:
        updated_x, updated_y = zip(*updated_obstacles)
    # Create a new figure
    plt.figure()

    # Plot the obstacle positions from each list
    #plt.scatter(prev_x, prev_y, color='red', label='Previous Obstacles', s=100)
    #plt.scatter(new_x, new_y, color='blue', label='New Obstacles', s=80)
    plt.scatter(updated_x, updated_y, color='green', label='Updated Obstacles')

    for obstacle, label in zip(updated_obstacles, updated_labels):
        plt.text(obstacle[0], obstacle[1], label, ha='center', va='bottom')  # Display the label near the obstacle

    # Add labels and legend
    plt.xlabel('X')
    plt.ylabel('Y')
    plt.xlim(0, 1.4)
    plt.ylim(0, 1.4)
    plt.legend()

    # Show the plot
    plt.show(block=False)

def calculate_distance(point1, point2):
    x1, y1 = point1
    x2, y2 = point2
    return math.sqrt((x2 - x1)**2 + (y2 - y1)**2)

def update_obstacles(previous_obstacles, new_obstacles, previous_labels, new_labels, camera_position, camera_orientation=0.2, fov=160, task="task1"):
    # Calculate FOV boundaries
    camera_orientation = math.degrees(camera_orientation)
    fov_half_angle = math.radians(fov / 2)
    fov_left_boundary = math.radians(camera_orientation) - fov_half_angle
    fov_right_boundary = math.radians(camera_orientation) + fov_half_angle

    # Initialize the updated obstacle positions list
    updated_obstacles = []
    updated_labels = []

    # Identify previous obstacles in the FOV
    obstacles_in_fov = []
    labels_in_fov = []
    for obstacle, label in zip(previous_obstacles, previous_labels):
        angle = math.atan2(obstacle[1] - camera_position[1], obstacle[0] - camera_position[0])
        angle = math.degrees(angle)
        angle_diff = (angle - camera_orientation + 180) % 360 - 180
        if -fov / 2 <= angle_diff <= fov / 2:
            obstacles_in_fov.append(obstacle)
            labels_in_fov.append(label)
    print('obstacles_in_fov', obstacles_in_fov)

    copy_obstacles_in_fov = copy.deepcopy(obstacles_in_fov)
    copy_new_obstacles = copy.deepcopy(new_obstacles)

    obstacles_in_fov_used = []
    new_obstacles_used = []
    previous_obstacles_used = []
    # Check if the number of obstacles in FOV is less than the number of new obstacles
    # Update obstacle positions for close obstacles
    for new_obstacle, new_label in zip(new_obstacles, new_labels):
        min_distance = 0.25
        closest_obstacle = None
        closest_label = None
        for obstacle in previous_obstacles:
            distance = math.sqrt((new_obstacle[0] - obstacle[0]) ** 2 + (new_obstacle[1] - obstacle[1]) ** 2)
            if distance < min_distance:
                min_distance = distance
                closest_obstacle = obstacle
                closest_label = label

        # Calculate the updated position as the average for close obstacles
        if closest_obstacle is not None and closest_obstacle not in previous_obstacles_used:
            if calculate_distance(camera_position, closest_obstacle) < 0.6:
                updated_position = new_obstacle
            else:
                updated_position = ((new_obstacle[0] + closest_obstacle[0]) / 2, (new_obstacle[1] + closest_obstacle[1]) / 2) # Calculate the updated position as the average
            #obstacles_in_fov.remove(closest_obstacle) # Remove the closest obstacle to avoid duplicate updating
            # labels_in_fov.remove(closest_label)
            updated_obstacles.append(updated_position) # Append the updated obstacle position to the updated list
            updated_labels.append(new_label)
            #add to used
            previous_obstacles_used.append(closest_obstacle)
            obstacles_in_fov_used.append(closest_obstacle)
            new_obstacles_used.append(new_obstacle)

    # Append the remaining new obstacles and previosu obstacles as updated obstacles that did not find a match
    updated_obstacles.extend(obstacle for obstacle in new_obstacles if obstacle not in new_obstacles_used)
    updated_labels.extend(label for label, obstacle in zip(new_labels, new_obstacles) if obstacle not in new_obstacles_used) #do you want to add previosu points that are in FOV but not used?
    updated_obstacles.extend(obstacle for obstacle in previous_obstacles if obstacle not in previous_obstacles_used)
    updated_labels.extend(label for label, obstacle in zip(previous_labels, previous_obstacles) if obstacle not in previous_obstacles_used)

    plot_obstacles(previous_obstacles, copy_new_obstacles, updated_obstacles, updated_labels)
    return updated_obstacles, updated_labels


import os, sys, argparse
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
import numpy as np
import pandas as pd

import tensorflow as tf
import tensorflow.keras.backend as K
from tensorflow.keras.layers import Add, Concatenate, Dense, Embedding, Flatten, Input, InputLayer, Lambda, Layer, Reshape, Subtract
tf.keras.backend.set_floatx('float64')

import util

elem_z = {
        'H'  : 1,
        'C'  : 6,
        'N'  : 7,
        'O'  : 8,
        'F'  : 9,
        'P'  : 15,
        'S'  : 16,
        'CL' : 17,
        'BR' : 35,
        }

if __name__ == '__main__':

    parser = argparse.ArgumentParser(description='Train an atomic multipole model')
    parser.add_argument('dataset_train',
                        help='Dataset for training (must be in ./data/)')
    parser.add_argument('dataset_val',
                        help='Dataset for validation (must be in ./data/)')
    parser.add_argument('modelname',
                        help='Name for saving model')
    args = parser.parse_args(sys.argv[1:])

    if os.path.isfile(f'models/{args.modelname}.hdf5'):
        print(f'Model models/{args.modelname}.hdf5 already exists!')
        exit()

    # these are defined by the dataset
    pad_dim = 40
    nelem = 36

    # this is up to the user
    nembed = 10
    nepochs = 200
    nnodes = [256,128,64]
    nmessage = 3

    # make the model
    mus = np.linspace(0.8, 5.0, 43)
    etas = np.array([-100.0] * 43)
    model = util.get_model(mus, etas, pad_dim, nelem, nembed, nnodes, nmessage)

    model.compile(optimizer=tf.keras.optimizers.Adam(0.001),
                  loss=['mse', util.mse_mp, util.mse_mp, util.mse_mp],
                  loss_weights=[1.0, 1.0, 1.0, 1.0],
                  metrics=[util.mae_mp])
    print(model.summary())

    # load data
    RT, ZT, yT = util.get_data(f'data/{args.dataset_train}.pkl', pad_dim)
    RV, ZV, yV = util.get_data(f'data/{args.dataset_val}.pkl', pad_dim)

    #RT, ZT, yT = RT[:800], ZT[:800], yT[:800]
    #RV, ZV, yV = RV[:800], ZV[:800], yV[:800]

    # monopole
    yV_ = yV[:,:,0]

    # dipole (mu_x, mu_y, mu_z)
    yV_i_ = yV[:,:,1:4]

    # quadrupole diagonal (Q_xx, Q_yy, Q_zz)
    yV_ii_ = yV[:,:,[4,7,9]]

    # quadrupole off-diagonal (Q_xy, Q_xz, Q_yz)
    yV_ij_ = yV[:,:,[5,6,8]]

    print('Validation Target Magnitudes (MAD):')
    for Z_subset, y_subset in [(ZV, [yV_, yV_i_, yV_ii_, yV_ij_])]:
        y_subset_pos = [ys[Z_subset > 0] for ys in y_subset]
        mad = [np.mean(np.abs(ys - np.mean(ys))) for ys in y_subset_pos]
        print(f'ALL ({y_subset_pos[0].shape[0]:6d}) : q({mad[0]:.4f}) mu({mad[1]:.4f}) Qii({mad[2]:.4f}) Qij({mad[3]:.4f})')
        for name, z in elem_z.items():
            mask = (Z_subset == z)
            y_element = [ys_[mask] for ys_ in y_subset]
            if np.sum(mask) > 0:
                mad = [np.mean(np.abs(ys - np.mean(ys))) for ys in y_element]
            else:
                mad = [np.nan for ys in y_element]
            print(f'{name:3s} ({np.sum(mask):6d}) : q({mad[0]:.4f}) mu({mad[1]:.4f}) Qii({mad[2]:.4f}) Qij({mad[3]:.4f})')
        print()

    print('Fitting Model...')
    callbacks = [tf.keras.callbacks.ModelCheckpoint(f'models/{args.modelname}.hdf5', save_best_only=True, monitor='val_loss', mode='min'),
                 tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=(10.0 ** (-1/4)), patience=10, verbose=1, mode='min', min_delta=0, cooldown=0, min_lr=(10.0 ** -5))]
    model.fit(x=util.RotationGenerator(RT, ZT, yT, batch_size=8),
              epochs=nepochs,
              validation_data=([RV, ZV], [yV_, yV_i_, yV_ii_, yV_ij_]),
              callbacks=callbacks,
              verbose=2)

    print('...Done')

    for w in model.get_layer(name='rbf').get_weights():
        print(w)

    yV_pred = model.predict([RV, ZV])

    yV_err = yV_pred[0] - yV_
    yV_i_err = yV_pred[1] - yV_i_
    yV_ii_err = yV_pred[2] - yV_ii_
    yV_ij_err = yV_pred[3] - yV_ij_

    print('Validation Prediction Magnitudes (MAE):')
    for Z_subset, y_subset in [(ZV, [yV_err, yV_i_err, yV_ii_err, yV_ij_err])]:
        y_subset_pos = [ys[Z_subset > 0] for ys in y_subset]
        mae = [np.mean(np.abs(ys)) for ys in y_subset_pos]
        print(f'ALL ({y_subset_pos[0].shape[0]:6d}) : q({mae[0]:.4f}) mu({mae[1]:.4f}) Qii({mae[2]:.4f}) Qij({mae[3]:.4f})')
        for name, z in elem_z.items():
            mask = (Z_subset == z)
            y_element = [ys_[mask] for ys_ in y_subset]
            if np.sum(mask) > 0:
                mae = [np.mean(np.abs(ys)) for ys in y_element]
            else:
                mae = [np.nan for ys in y_element]
            print(f'{name:3s} ({np.sum(mask):6d}) : q({mae[0]:.4f}) mu({mae[1]:.4f}) Qii({mae[2]:.4f}) Qij({mae[3]:.4f})')
        print()