import math
import random
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
from collections import namedtuple, deque
from itertools import count
from random import choice
from copy import deepcopy

from project.tools.logger import logger, init_logger

import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F

from project.env.environnement import Environnement
from project.env.states import State

env = Environnement.init("3dadvanced")

# set up matplotlib
is_ipython = 'inline' in matplotlib.get_backend()
if is_ipython:
    from IPython import display

plt.ion()

# if gpu is to be used
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

Transition = namedtuple('Transition',
                        ('state', 'action', 'next_state', 'reward'))

class ReplayMemory(object):

    def __init__(self, capacity):
        self.memory = deque([],maxlen=capacity)

    def push(self, *args):
        """Save a transition"""
        self.memory.append(Transition(*args))

    def sample(self, batch_size):
        return random.sample(self.memory, batch_size)

    def __len__(self):
        return len(self.memory)

class DQN(nn.Module):

    def __init__(self, n_observations, n_actions):
        super(DQN, self).__init__()
        self.layer1 = nn.Linear(n_observations, 256)
        self.layer2 = nn.Linear(256, 2048)
        self.layer3 = nn.Linear(2048, 128)
        self.layer4 = nn.Linear(128, n_actions)

    # Called with either one element to determine next action, or a batch
    # during optimization. Returns tensor([[left0exp,right0exp]...]).
    def forward(self, x):
        x = F.relu(self.layer1(x))
        x = F.relu(self.layer2(x))
        x = F.relu(self.layer3(x))
        return self.layer4(x)

# BATCH_SIZE is the number of transitions sampled from the replay buffer
# GAMMA is the discount factor as mentioned in the previous section
# EPS_START is the starting value of epsilon
# EPS_END is the final value of epsilon
# EPS_DECAY controls the rate of exponential decay of epsilon, higher means a slower decay
# TAU is the update rate of the target network
# LR is the learning rate of the AdamW optimizer
BATCH_SIZE = 64
GAMMA = 0.9999
EPS_START = 0.1
EPS_END = 0.005
EPS_DECAY = 10000
TAU = 0.005
LR = 1e-3

init_logger(logger)

all_poss_action = env.getPossibleActions()
all_poss_state = env.getEveryState()
# ennumerate all state
transfo_state_to_int = dict()
for i,x in enumerate(all_poss_state):
    transfo_state_to_int[x] = i
# Get number of actions from gym action space
n_actions = len(all_poss_action)
# Get the number of state observations
transfo_action_to_int = dict()
for i,a in enumerate(all_poss_action):
    transfo_action_to_int[a] = i

def one_hot_state(s:int):
    res = np.zeros(len(all_poss_state))
    res[s] = 1
    return res

def one_hot_action(a:int):
    res = np.zeros(len(all_poss_action))
    res[a] = 1
    return res

state = env.reset()

def state_to_obs(state:State):
    np_obs = one_hot_state(transfo_state_to_int[state])
    return torch.tensor(np_obs, device=device, dtype=torch.float32).unsqueeze(0)

n_observations = 35

policy_net = DQN(n_observations, n_actions).to(device)
target_net = DQN(n_observations, n_actions).to(device)
target_net.load_state_dict(policy_net.state_dict())

optimizer = optim.AdamW(policy_net.parameters(), lr=LR, amsgrad=True)
memory = ReplayMemory(10000)

steps_done = 0

def select_action(state):
    global steps_done
    sample = random.random()
    eps_threshold = EPS_END + (EPS_START - EPS_END) * \
        math.exp(-1. * steps_done / EPS_DECAY)
    steps_done += 1
    if sample > eps_threshold:
        with torch.no_grad():
            # t.max(1) will return largest column value of each row.
            # second column on max result is index of where max element was
            # found, so we pick action with the larger expected reward.
            return policy_net(state).max(1)[1]
    else:
        return torch.tensor([transfo_action_to_int[choice(all_poss_action)]], device=device, dtype=torch.long)

rewards = []

def plot_rewards(show_result=False):
    plt.figure(1)
    rewards_t = torch.tensor(rewards, dtype=torch.float)
    if show_result:
        plt.title('Result')
    else:
        plt.clf()
        plt.title('Training...')
    plt.xlabel('Episode')
    plt.ylabel('Reward')
    plt.plot(rewards_t.numpy())
    # Take 100 episode averages and plot them too
    if len(rewards_t) >= 100:
        means = rewards_t.unfold(0, 100, 10).mean(10).view(-1)
        means = torch.cat((torch.zeros(99), means))
        plt.plot(means.numpy())

    plt.pause(0.001)  # pause a bit so that plots are updated
    if is_ipython:
        if not show_result:
            display.display(plt.gcf())
            display.clear_output(wait=True)
        else:
            display.display(plt.gcf())

def optimize_model():
    if len(memory) < BATCH_SIZE:
        return
    transitions = memory.sample(BATCH_SIZE)
    # Transpose the batch (see https://stackoverflow.com/a/19343/3343043 for
    # detailed explanation). This converts batch-array of Transitions
    # to Transition of batch-arrays.
    batch = Transition(*zip(*transitions))

    # Compute a mask of non-final states and concatenate the batch elements
    # (a final state would've been the one after which simulation ended)
    non_final_mask = torch.tensor(tuple(map(lambda s: s is not None,
                                          batch.next_state)), device=device, dtype=torch.bool)
    non_final_next_states = torch.cat([s for s in batch.next_state
                                                if s is not None])
    state_batch = torch.cat(batch.state)
    action_batch = torch.cat(batch.action)
    reward_batch = torch.cat(batch.reward)

    # Compute Q(s_t, a) - the model computes Q(s_t), then we select the
    # columns of actions taken. These are the actions which would've been taken
    # for each batch state according to policy_net
    state_action_values = policy_net(state_batch).gather(1, action_batch)

    # Compute V(s_{t+1}) for all next states.
    # Expected values of actions for non_final_next_states are computed based
    # on the "older" target_net; selecting their best reward with max(1)[0].
    # This is merged based on the mask, such that we'll have either the expected
    # state value or 0 in case the state was final.
    next_state_values = torch.zeros(BATCH_SIZE, device=device)
    with torch.no_grad():
        next_state_values[non_final_mask] = target_net(non_final_next_states).max(1)[0]
    # Compute the expected Q values
    expected_state_action_values = (next_state_values * GAMMA) + reward_batch

    # Compute Huber loss
    criterion = nn.MSELoss()
    loss = criterion(state_action_values, expected_state_action_values.detach())

    # Optimize the model
    optimizer.zero_grad()
    loss.backward()
    # In-place gradient clipping
    torch.nn.utils.clip_grad_value_(policy_net.parameters(), 100)
    optimizer.step()

num_episodes = 2000

for i_episode in range(num_episodes):
    # Initialize the environment and get it's state
    state = env.reset()
    state = state_to_obs(state)
    done = False
    while not done:
        action = F.one_hot(select_action(state), n_actions)

        observation, reward, done = env.step(all_poss_action[torch.argmax(action)])
        reward = torch.tensor([reward], device=device)
        logger.info(f"Step : {env.step_number-1} Agent observes: state={all_poss_state[torch.argmax(state)]}, action={all_poss_action[torch.argmax(action)]}, reward={reward}, done={done}")

        env.render()

        if done:
            print(state)
            print(env.score)
            if i_episode%2 == 1:
                env.final_render()
            next_state = None
        else:
            next_state = state_to_obs(observation)

        # Store the transition in memory
        memory.push(state, action, next_state, reward)

        # Move to the next state
        state = deepcopy(next_state)

        # Perform one step of the optimization (on the policy network)
        optimize_model()

        # Soft update of the target network's weights
        # θ′ ← τ θ + (1 −τ )θ′
        target_net_state_dict = target_net.state_dict()
        policy_net_state_dict = policy_net.state_dict()
        for key in policy_net_state_dict:
            target_net_state_dict[key] = policy_net_state_dict[key]*TAU + target_net_state_dict[key]*(1-TAU)
        target_net.load_state_dict(target_net_state_dict)

        if done:
            print(EPS_END + (EPS_START - EPS_END) * \
        math.exp(-1. * steps_done / EPS_DECAY))
            rewards.append(env.score)
            plot_rewards()
            break

print('Complete')
plot_rewards(show_result=True)
plt.ioff()
plt.show()


import tensorflow as tf
from  sklearn.datasets import load_boston

data = load_boston()

import pandas as pd
df = pd.DataFrame(data.data)

df_ = pd.DataFrame(data.target)
df__ =pd.concat([df,df_],axis=1)
print(df__)

model =tf.keras.Sequential()
model.add(tf.keras.layers.Dense(10,input_shape=(13,),activation=tf.keras.activations.relu))
model.add(tf.keras.layers.Dense(5,activation=tf.keras.activations.relu))
model.add(tf.keras.layers.Dense(1))

'''

梯度衰减
https://blog.csdn.net/Light2077/article/details/106629697?utm_medium=distribute.pc_relevant.none-task-blog-baidujs-1

  return initial_learning_rate * decay_rate ^ (step / decay_steps)

'''
# 学习率衰减
exponential_decay = tf.keras.optimizers.schedules.ExponentialDecay(
                        initial_learning_rate=0.1, decay_steps=1000, decay_rate=0.96,staircase=True)

model.compile(optimizer=tf.keras.optimizers.Adam(exponential_decay),loss=tf.keras.losses.mse)
history = model.fit(data.data,data.target,epochs=300)

pre = model.predict(df__.iloc[0:10,0:-1])

print(pre)

print(df__.iloc[0:10,-1])


#!/usr/bin/env python

from xmi2odoo import uml

def tag_option(obj, name, label=None, default=None, check=True, quote='\'', negate=False, translate=False):
    if isinstance(name, list):
        valid = any(n in obj.tag for n in name)
        label = label or name[0]
        value = valid and ("%s,%s" % quote).join(obj.tag[n] for n in name if n in obj.tag)
    else:
        valid = name in obj.tag
        label = label or name
        value = valid and obj.tag[name]
    pretrans = "_(" if translate else ""
    posttrans = ")" if translate else ""
    if not(label=='string' and obj.tag[name]==obj.name):
        if check and (not valid if negate else valid):
            r = "%s=%s%s%s%s%s" % (label, pretrans, quote, obj.tag[name], quote, posttrans)
        else:
            r = default
    else:
        r = default
    return r

def stereotype_option(obj, name, label=None, default=None, value='True', check=True, negate=False):
    label = label or name
    valid = obj.is_stereotype(name)
    if check and (not valid if negate else valid):
        r = "%s=%s" % (label, value)
    else:
        r = default
    return r

def model(obj, s='.'):
    try:
        return obj.oerp_id(s, False)
    except:
        return obj.oerp_id(s)

def emodel(obj, s='.'):
    return obj.oerp_id(s)

def pmodel(obj, s='.'):
    return obj.oerp_id(s, False, True)

def name(obj, prefix='', suffix='', default=''):
    return "%s%s%s" % (prefix, obj.name if obj.name not in [ None, '' ] else default, suffix)

def names(obj, prefix='', suffix='', default=''):
    return [ name(o, prefix=prefix, suffix=suffix, default=default) for o in obj ]

def attr_options(cls, obj, version=False):
    if version == '8.0':
      return ',\n        '.join([ o for o in [
         tag_option(obj,  'selection', quote=''),
         tag_option(obj,  'model_name'),
         tag_option(obj,  'relation'),
         tag_option(obj,  'comodel_name'),
         tag_option(obj,  'column1'),
         tag_option(obj,  'column2'),
         tag_option(obj,  'inverse_name'),
         tag_option(obj,  'label', label='string'),
         tag_option(obj,  'documentation', label='help', quote='"""'),
         tag_option(obj,  'ondelete', quote=''),
         tag_option(obj,  'track_visibility'),
         tag_option(obj,  'digits',quote=''),
         stereotype_option(obj, 'readonly'),
         stereotype_option(obj, 'required', check=not cls.is_extended()),
         tag_option(obj,  'size'),
         tag_option(obj,  'states', quote=''),
         tag_option(obj,  'context', quote=''),
         tag_option(obj,  'domain', quote=''),
         tag_option(obj,  'on_change'),
         tag_option(obj,  ['groups', 'module_groups'], label='groups'),
         stereotype_option(obj, 'change_default'),
         stereotype_option(obj, 'select'),
         stereotype_option(obj, 'store'),
         stereotype_option(obj, 'translatable', label='translate'),
         stereotype_option(obj, 'invisible'),
         stereotype_option(obj, 'relation',value=obj.datatype.oerp_id()),
         stereotype_option(obj, 'method'),
         stereotype_option(obj, 'view_load'),
         stereotype_option(obj, 'group_name',value=cls.tag.get('group',cls.package.tag['label'])),
         tag_option(obj,  'default', quote=''),
         tag_option(obj,  'fnct', label='compute'),
         tag_option(obj,  'fnct_inv', label='inverse'),
         tag_option(obj,  'fnct_search', label='search'),
         tag_option(obj,  'related'),
         tag_option(obj,  'related_to', label='related'),
         tag_option(obj,  'compute'),
         tag_option(obj,  'inverse'),
         tag_option(obj,  'search'),
         tag_option(obj,  'copy', quote=''),
         ] if o is not None ])
    return ','.join([ o for o in [
       tag_option(obj,  'label', label='string'),
       tag_option(obj,  'documentation', label='help', quote='"""'),
       tag_option(obj,  'ondelete', quote=''),
       tag_option(obj,  'digits', quote=''),
       stereotype_option(obj, 'readonly'),
       stereotype_option(obj, 'required', check=not cls.is_extended()),
       tag_option(obj,  'size'),
       tag_option(obj,  'states',),
       tag_option(obj,  'context'),
       tag_option(obj,  'domain'),
       tag_option(obj,  'on_change'),
       tag_option(obj,  ['groups', 'module_groups'], label='groups'),
       stereotype_option(obj, 'change_default'),
       stereotype_option(obj, 'select'),
       stereotype_option(obj, 'store'),
       stereotype_option(obj, 'translatable', label='translate'),
       stereotype_option(obj, 'invisible'),
       stereotype_option(obj, 'relation',value=obj.datatype.oerp_id()),
       stereotype_option(obj, 'method'),
       stereotype_option(obj, 'view_load'),
       stereotype_option(obj, 'group_name',value=cls.tag.get('group',cls.package.tag['label'])),
       tag_option(obj,  'fnc_inv'),
       tag_option(obj,  'fnc_search'),
       ] if o is not None ])

def class_id(CLASS):
    return "%s_id" % CLASS.name

def ass_id(obj):
    return obj.participant.oerp_id()

def ass_other_name(cls, obj):
    return "%s_id" % cls.name if obj.swap[0].name in [None, ''] else obj.swap[0].name

def ass_other_id(ass):
    return "%s_id" % ass.participant.name.replace('.','_')

def ass_relational_obj(mod, ass):
    return '%s_%s_rel' % (mod,
                          '_'.join([ e.name or '_'
                                    for e in ass.association.ends]))

def ass_options(cls,obj,version=False):
    if version == '8.0':
      return ',\n        '.join([ o for o in [
        tag_option(obj,  'selection', quote=''),
        tag_option(obj,  'model_name'),
        tag_option(obj,  'relation'),
        tag_option(obj,  'comodel_name'),
        tag_option(obj,  'column1'),
        tag_option(obj,  'column2'),
        tag_option(obj,  'inverse_name'),
        tag_option(obj,  'label', label='string'),
        tag_option(obj,  'documentation', label='help', quote='"""'),
        tag_option(obj,  'ondelete'),
        tag_option(obj,  'track_visibility'),
        tag_option(obj,  'digits'),
        stereotype_option(obj, 'readonly'),
        stereotype_option(obj, 'required', check=(not cls.is_extended())),
        stereotype_option(obj, 'required', check=(not cls.is_extended()) and ((eval(obj.multiplicityrange) or (0,0))[0] > 0), negate=True),
        tag_option(obj,  'size'),
        tag_option(obj,  'states', quote=''),
        tag_option(obj,  'context', quote=''),
        tag_option(obj,  'domain', quote=''),
        tag_option(obj,  'on_change'),
        tag_option(obj,  ['groups', 'module_groups'], label='groups'),
        stereotype_option(obj, 'change_default'),
        stereotype_option(obj, 'select'),
        stereotype_option(obj, 'store'),
        stereotype_option(obj, 'translatable', label='translate'),
        stereotype_option(obj, 'invisible'),
        #stereotype_option(obj, 'relation',value=obj.datatype.oerp_id()),
        stereotype_option(obj, 'method'),
        stereotype_option(obj, 'view_load'),
        stereotype_option(obj, 'group_name',value=cls.tag.get('group',cls.package.tag['label'])),
        tag_option(obj,  'fnc_inv'),
        tag_option(obj,  'fnc_search'),
        tag_option(obj,  'default', quote=''),
        tag_option(obj,  'fnct', label='compute'),
        tag_option(obj,  'fnct_inv', label='inverse'),
        tag_option(obj,  'fnct_search', label='search'),
        tag_option(obj,  'related'),
        tag_option(obj,  'related_to', label='related'),
        tag_option(obj,  'compute'),
        tag_option(obj,  'inverse'),
        tag_option(obj,  'search'),
        tag_option(obj,  'copy', quote=''),
        ] if o is not None ])
    return ','.join([ o for o in [
       tag_option(obj,  'label', label='string', translate=True),
       tag_option(obj,  'documentation', label='help', quote='"""'),
       tag_option(obj,  'ondelete'),
       tag_option(obj,  'digits'),
       stereotype_option(obj, 'readonly'),
       stereotype_option(obj, 'required', check=(not cls.is_extended())),
       stereotype_option(obj, 'required', check=(not cls.is_extended()) and ((eval(obj.multiplicityrange) or (0,0))[0] > 0), negate=True),
       tag_option(obj,  'size'),
       tag_option(obj,  'states',),
       tag_option(obj,  'context',),
       tag_option(obj,  'domain',),
       tag_option(obj,  'on_change'),
       tag_option(obj,  ['groups', 'module_groups'], label='groups'),
       stereotype_option(obj, 'change_default'),
       stereotype_option(obj, 'select'),
       stereotype_option(obj, 'store'),
       stereotype_option(obj, 'translatable', label='translate'),
       stereotype_option(obj, 'invisible'),
       #stereotype_option(obj, 'relation',value=obj.datatype.oerp_id()),
       stereotype_option(obj, 'method'),
       stereotype_option(obj, 'view_load'),
       stereotype_option(obj, 'group_name',value=cls.tag.get('group',cls.package.tag['label'])),
       tag_option(obj,  'fnc_inv'),
       tag_option(obj,  'fnc_search'),
       ] if o is not None ])

def sel_literals(col):
    return repr([(i.name, i.tag.get('label',i.name)) for i in col.datatype.all_literals()]) if col.datatype.not_is_stereotype('function') else "_get_%s" % col.datatype.name

def fnc_name(col):
    return name(col, prefix='property_' if col.is_stereotype('property') else '')

def parameters(op, prefix='', suffix=''):
    parms = [par.name for par in [ p for p in op.parameters if p.name != 'return']]
    if parms:
        return '%s%s%s' % (prefix, ', '.join(parms), suffix)
    else:
        return ''

def view_filter_id(cls):
    return "view_%s_filter" % model(cls, '_')

def view_form_id(cls):
    return "view_%s_form" % model(cls, '_')

def view_tree_id(cls):
    return "view_%s_tree" % model(cls, '_')

def form_colors(cls):
    r = ''
    for sm in [ sm for sm in cls.get_statemachines(no_stereotypes=['extend','prototype'])]:
        r = "grey:state=='cancelled';blue:state in %s;black:state in %s;red:state in %s" % (
            repr(tuple(names(sm.initial_states()))),
            repr(tuple(set(names(sm.middle_states()))-set(names(sm.stereotype_states('exception'))))),
            repr(tuple(names(sm.stereotype_states('exception'))))
        )
    return r

def tree_sufix(menu):
    return (menu.is_stereotype('hierarchical') and '_hier' or '') + (menu.is_stereotype('editable') and '_edit' or '')

def wkf_name(cls, obj):
    return "%s_%s_wkf" % (cls.name, obj.name)

def wkf_guard(cls, u=' and '):
    if cls.oerp_id('-', False) != cls.oerp_id('-') and cls.has_member('type', uml.CAttribute):
        r = 'type =="%s"%s' % (model(cls), u)
    else:
        r = 'True%s' % u
    return r

def groups(obj):
    return ','.join([ '(4, ref(\'%s\'))' % (name if '.' in name else 'group_%s' % name) for name in obj.tag['groups'].split(',')])

def is_related(obj):
    return obj.relateds(uml.CClass)

def related(obj):
    assert list(obj.relateds(uml.CClass)) > 1, "You have more than one class related to %s.\n%s" % (obj.name, names(obj.relateds(uml.CClass)))
    return obj.relateds(uml.CClass)[0]

def walk_by_associations(CLASS, related_by):
    if related_by:
        relation = CLASS.association_by_name(related_by[0])
        if relation:
            return walk_by_associations(relation.participant, related_by[1:])
        else:
            return None
    return CLASS

def debug(*values):
    import pdb; pdb.set_trace()
    return values


from ripple_heterogeneity.utils import functions, loading, add_new_deep_sup
from ripple_heterogeneity.assembly import assembly_reactivation,find_sig_assembly
from ripple_heterogeneity.readout import assembly_multi_region
import pandas as pd
import numpy as np
import os
import nelpy as nel
import glob
import pickle
import pynapple as nap
import logging

logging.getLogger().setLevel(logging.ERROR)

def locate_task_epoch(epoch_df, env):
    epoch_df = epoch_df.reset_index(drop=True).copy()
    epoch_df["duration"] = epoch_df.stopTime - epoch_df.startTime
    epoch_df.sort_values("duration", ascending=False, inplace=True)
    return int(epoch_df[epoch_df.environment.str.contains(env)].index[0])

def get_pos(basepath, epochs, epoch_df, task_idx):

    position_df = loading.load_animal_behavior(basepath)

    if "linearized" not in position_df.columns:
        return None, None, None, None

    position_df_no_nan = position_df.query(
        "not x.isnull() & not y.isnull() & not linearized.isnull()"
    )

    if position_df_no_nan.shape[0] == 0:
        return None, None, None, None

    pos = nel.PositionArray(
        data=position_df_no_nan["linearized"].values.T,
        timestamps=position_df_no_nan.timestamps.values,
    )
    # make min pos 2
    pos._data = (pos.data - np.nanmin(pos.data)) + 2

    pos = pos[epochs[task_idx]]

    # get outbound and inbound epochs
    outbound_epochs, inbound_epochs = functions.get_linear_track_lap_epochs(
        pos.abscissa_vals, pos.data[0], newLapThreshold=20
    )

    position_df_no_nan = position_df_no_nan[
        position_df_no_nan["time"].between(
            epoch_df.iloc[task_idx].startTime, epoch_df.iloc[task_idx].stopTime
        )
    ]
    return pos, outbound_epochs, inbound_epochs, position_df_no_nan

def get_cross_region_assemblies(
    basepath,
    regions=None,  # brain regions to load
    deepSuperficial=None,
    putativeCellType="Pyr",  # type of cells to load (can be multi ex. Pyr|Int)
    weight_dt=0.05,  # dt in seconds for binning st to get weights for each assembly
    z_mat_dt=0.03,
    epoch=None,
):
    m1 = assembly_reactivation.AssemblyReact(
        basepath,
        brainRegion=regions[0] + "|" + regions[1],
        putativeCellType=putativeCellType,
        weight_dt=weight_dt,
        z_mat_dt=z_mat_dt,
    )
    st, cell_metrics = loading.load_spikes(
        basepath,
        brainRegion=regions[0] + "|" + regions[1],
        putativeCellType=putativeCellType,
    )
    cell_metrics = add_new_deep_sup.deep_sup_from_deepSuperficialDistance(cell_metrics)
    m1.st = st.iloc[
        :,
        cell_metrics.deepSuperficial.str.contains(deepSuperficial + "|unknown").values,
    ]
    m1.cell_metrics = cell_metrics[
        cell_metrics.deepSuperficial.str.contains(deepSuperficial + "|unknown").values
    ]
    # check for any cortex cells
    if (
        not m1.cell_metrics.brainRegion.str.contains(regions[0]).any()
        & m1.cell_metrics.brainRegion.str.contains(regions[1]).any()
    ):
        return None, None

    m1.get_weights(epoch=epoch)
    if len(m1.patterns) == 0:
        return None, None

    assembly_act = m1.get_assembly_act(epoch=epoch)
    return m1, assembly_act

def run(
    basepath,
    regions="CA1|PFC|EC1|EC2|EC3|EC4|EC5|MEC",  # brain regions to load
    cross_regions=(("CA1", "PFC"), ("CA1", "EC1|EC2|EC3|EC4|EC5|MEC")),
    putativeCellType="Pyr",  # type of cells to load (can be multi ex. Pyr|Int)
    weight_dt=0.05,  # dt in seconds for binning st to get weights for each assembly
    z_mat_dt=0.03,  # dt in seconds for binning st to get activation strength
    verbose=False,  # print out progress
    env="linear",  # enviroment you want to look at (current should only be linear)
    s_binsize=3,  # spatial bin size
    smooth_sigma=3,  # smoothing sigma in cm
    smooth_window=10,  # smoothing window in cm
    speed_thres=4,  # speed threshold for ratemap in cm/sec
):

    epoch_df = loading.load_epoch(basepath)
    epochs = nel.EpochArray(np.array([epoch_df.startTime, epoch_df.stopTime]).T)

    # check if any env
    if not epoch_df.environment.str.contains(env).any():
        return None

    # find longest xx session
    # task_idx = locate_task_epoch(m1, env)
    # position_df = loading.load_animal_behavior(basepath)
    # task_idx = int(np.where(epoch_df.name == position_df[~position_df.linearized.isnull()].epochs.unique()[0])[0][0])

    # find longest xx session
    task_idx = locate_task_epoch(epoch_df, env)

    # get cross region assemblies
    assem_labels = []
    assembly_act = []
    abscissa_vals = []
    m1 = {}
    for cross_region in cross_regions:
        for deepSuperficial in ["Deep", "Superficial"]:
            m1_, assembly_act_ = get_cross_region_assemblies(
                basepath,
                regions=cross_region,
                deepSuperficial=deepSuperficial,
                putativeCellType=putativeCellType,
                weight_dt=weight_dt,
                z_mat_dt=z_mat_dt,
                epoch=epochs[task_idx],
            )
            if assembly_act_ is None:
                continue
            if m1_.n_assemblies() == 0:
                continue

            # make sure assembly members represent cross region label
            keep_assembly = []
            _, _, keep_assembly_, is_member = find_sig_assembly.main(m1_.patterns)
            for assembly_i in range(m1_.n_assemblies()):
                member_idx = is_member[assembly_i, :]
                cortex_check = (
                    m1_.cell_metrics[member_idx]
                    .brainRegion.str.contains(cross_region[1])
                    .any()
                )
                ca1_check = (
                    m1_.cell_metrics[member_idx]
                    .brainRegion.str.contains(cross_region[0])
                    .any()
                )
                ca1_layer_check = (
                    m1_.cell_metrics[member_idx].deepSuperficial == deepSuperficial
                ).any()
                keep_assembly.append(cortex_check & ca1_check & ca1_layer_check)

            assembly_act.append(assembly_act_.data[keep_assembly, :])
            abscissa_vals.append(assembly_act_.abscissa_vals)
            assem_labels.append([deepSuperficial + "_" + cross_region[1]] * sum(keep_assembly))
            m1[deepSuperficial + "_" + cross_region[1]] = m1_

    if len(assembly_act) == 0:
        return None

    assembly_act = nel.AnalogSignalArray(
        data=np.vstack(assembly_act),
        timestamps=abscissa_vals[0],
    )
    if assembly_act.isempty:
        return None

    assem_labels = np.hstack(assem_labels)

    pos, outbound_epochs, inbound_epochs, position_df_no_nan = get_pos(
        basepath, epochs, epoch_df, task_idx
    )
    if pos is None:
        return None

    ext_xmin, ext_xmax = (
        np.floor(pos.data[0].min() / 10) * 10,
        np.ceil(pos.data[0].max() / 10) * 10,
    )
    n_extern = int((ext_xmax - ext_xmin) / s_binsize)

    label_df = pd.DataFrame()
    tc = pd.DataFrame()

    direction_label = ["outbound_epochs", "inbound_epochs"]

    for dir_epoch_i, dir_epoch in enumerate([outbound_epochs, inbound_epochs]):

        # compute and smooth speed
        speed1 = nel.utils.ddt_asa(pos[dir_epoch], smooth=True, sigma=0.1, norm=True)

        # find epochs where the animal ran > 4cm/sec
        run_epochs = nel.utils.get_run_epochs(speed1, v1=speed_thres, v2=speed_thres)

        # add assembly activations to tsdframe object
        tsdframe = nap.TsdFrame(
            t=assembly_act[dir_epoch][run_epochs].abscissa_vals,
            d=assembly_act[dir_epoch][run_epochs].data.T,
        )

        # add position to tsd object
        feature = nap.Tsd(
            t=pos[dir_epoch][run_epochs].abscissa_vals,
            d=pos[dir_epoch][run_epochs].data[0],
        )

        # compute tuning curves
        tc_ = nap.compute_1d_tuning_curves_continous(
            tsdframe, feature, nb_bins=n_extern, minmax=[ext_xmin, ext_xmax]
        )

        # smooth
        tc_ = tc_.rolling(
            window=smooth_window, win_type="gaussian", center=True, min_periods=1
        ).mean(std=smooth_sigma)

        # store tuning curves
        tc = pd.concat([tc, tc_], axis=1, ignore_index=True)

        # store associated metadata
        label_df_ = pd.DataFrame()
        label_df_["assembly_n"] = np.arange(assembly_act.n_signals).astype(int)
        label_df_["direction_label"] = direction_label[dir_epoch_i]
        label_df_["cross_region_label"] = assem_labels

        label_df = pd.concat([label_df, label_df_], ignore_index=True)

    label_df["basepath"] = basepath

    results = {
        "tc": tc,
        "label_df": label_df,
        "assembly_act_task": assembly_act,
        "react": m1,
        "pos": pos,
        "outbound_epochs": outbound_epochs,
        "inbound_epochs": inbound_epochs,
        "task_epoch": epochs[task_idx],
    }

    return results

def load_results(save_path, verbose=False):
    """
    load_results: load results from a pickle file
    """
    print("Loading results...")

    sessions = glob.glob(save_path + os.sep + "*.pkl")

    tc = []
    label_df = pd.DataFrame()

    for session in sessions:
        if verbose:
            print(session)
        with open(session, "rb") as f:
            results = pickle.load(f)
        if results is None:
            continue

        tc.append(results["tc"])
        label_df = pd.concat([label_df, results["label_df"]], ignore_index=True)

    return tc, label_df


from setuptools import setup
import os
from glob import glob

package_name = 'cr3_scout_with_srvs'

setup(
    name=package_name,
    version='0.0.0',
    packages=[package_name],
    data_files=[
        #(os.path.join('share',package_name,'launch'), glob('launch/*_launch.py')),
        (os.path.join('share',package_name,'launch'), glob('launch/*_launch.py')),
        (os.path.join('share',package_name,'config'), glob('config/*.yaml')),
        ('share/ament_index/resource_index/packages',
            ['resource/' + package_name]),
        ('share/' + package_name, ['package.xml']),
    ],
    install_requires=['setuptools'],
    zip_safe=True,
    maintainer='qiuyuezuo',
    maintainer_email='qiuyuezuo@todo.todo',
    description='TODO: Package description',
    license='TODO: License declaration',
    tests_require=['pytest'],
    entry_points={
        'console_scripts': [
            'scout = cr3_scout_with_srvs.scout:main',
            'client = cr3_scout_with_srvs.cr3_service_clients:main',
            'scout_with_zoom_camera = cr3_scout_with_srvs.scout_with_zoom_camera:main',
            'camera_intrin_info = cr3_scout_with_srvs.camera_intrin_info:main',
            'scout_with_zoom_camera_horiz = cr3_scout_with_srvs.scout_with_zoom_camera_horiz:main',
            #'cam_info_fake_pub = cr3_scout_with_srvs.cam_info_fake_pub:main'
        ],
    },
)


import random

import torch
from models.wgan_model import WGanModel
from collections import defaultdict
from tqdm import tqdm
from torchvision.utils import make_grid
import math
from torch.utils.tensorboard import SummaryWriter
from torch import optim
import inspect
from models import find_model_using_name
from torch.autograd import grad, Variable

from trainers.base_trainer import BaseTrainer

class WGanTrainer(BaseTrainer):
    """
    BaseTrainer receives the options and initialize optimizers, models, losses, iters, D(x), num_epochs
    """

    def __init__(self, opt, iters_per_epoch=math.inf):
        super().__init__(opt, iters_per_epoch)
        self.fix_noise = torch.rand(opt.num_display_images, opt.noise_dim, 1, 1)
        # self.opt.name += f'_{self.model.clipping_limit}'
        # self.fix_noise.requires_grad = True
        # print(f'fix noise required grad = {self.fix_noise.requires_grad}')

    def _cal_dis_grad(self, real_data, fake_data):
        alpha = torch.rand(real_data.shape[0], 1, 1, 1).expand_as(real_data).to(real_data.device, non_blocking=True)
        max_data = Variable(alpha * real_data + (1 - alpha) * fake_data, requires_grad=True)
        mix_logits = self.model.netD(max_data)
        mix_grad = grad(mix_logits, max_data, grad_outputs=torch.ones_like(mix_logits))[0]
        # mix_logits.backward(torch.ones_like(mix_logits))
        # mix_grad = max_data.grad
        return mix_grad.max()
        # return mean(fake_grad)

    @torch.no_grad()
    def _generate_image(self):
        # print(f'inside generate_image')
        # print(f'fix noise required grad = {self.fix_noise.requires_grad}')
        fake_data = self.model.netG(self.fix_noise)
        # print(f'fake_data required grad = {fake_data.requires_grad}')
        # print(fake_data.min(), fake_data.max())
        fake_data = (fake_data / 2 + 0.5).detach().cpu()
        # print(fake_data.min(), fake_data.max())
        # exit()
        return fake_data

    def _write_tf_log(self, writer, epoch):
        # for losses
        for key, value in self.losses.items():
            scalar = sum(value) / len(value)
            writer.add_scalar(f'Loss/{key}', scalar, epoch)
        # for discriminator outputs
        writer.add_scalars(f'D(x)', {key: sum(value) / len(value)
                                     for key, value in self.dis_outputs.items()
                                     }, epoch)
        # for generated image
        generated_images = self._generate_image()
        # print(generated_images.shape)
        nrow = int(math.sqrt(self.opt.num_display_images))
        image_grid = make_grid(generated_images, nrow=nrow)
        writer.add_image('Generated Image', image_grid, epoch)

    def train(self, train_loader, val_loader=None):
        """
        epoch start with 1, end with num_epochs
        """
        writer = SummaryWriter(self.opt.log_dir / self.opt.name)
        for epoch in range(1, self.opt.num_epochs + 1):
            self.losses.clear()
            self.dis_outputs.clear()
            self._train_epoch(train_loader, epoch)
            if val_loader is not None:
                self._val_epoch(val_loader)
            self._write_tf_log(writer, epoch)
            if epoch % self.opt.save_ckpt_freq == 0:
                self.model.save(epoch)
        writer.close()

    def _train_epoch(self, data_loader, epoch):
        pbar = tqdm(data_loader, colour='MAGENTA')
        # BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE
        for batch_data in pbar:
            self.iters += 1
            pbar.set_description(f'Epoch: [{epoch}/{self.opt.num_epochs}], '
                                 f'Iter: [{self.iters}/{self.opt.num_iters}]')
            # print(batch_data.min(), batch_data.max())
            batch_data = batch_data.to(self.opt.device, non_blocking=True)
            self._train_discriminator_once(batch_data)
            if self.iters % self.opt.num_critics == 0:
                self._train_generator_once(batch_data.shape[0])
            if self.iters % self.opt.save_latest_freq == 0:
                self.model.save('latest')
            pbar.set_postfix(w_dis=f'{-sum(self.losses["gan_D"]) / len(self.losses["gan_D"]):.4f}',
                             g_loss=f'{sum(self.losses["gan_G"]) / (len(self.losses["gan_G"]) + 1e-12):.4f}')
            # ,
            # dis_grad=f'{max(self.losses["dis_grad"]):.4f}')
        for model_name in self.schedulers.keys():
            self.schedulers[model_name].step()

    @torch.no_grad()
    def _val_epoch(self, data_loader):
        pbar = tqdm(data_loader, colour='MAGENTA')
        # BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE
        for batch_data in pbar:
            pbar.set_description('Validating... ')
            batch_data = batch_data.to(self.opt.device, non_blocking=True)
            val_logits = self.model.netD(batch_data)
            self.dis_outputs['val'] += val_logits.flatten().tolist()

    def _train_generator_once(self, batch_size):
        self.optimizers['G'].zero_grad()
        fake_data = self.model.netG(batch_size)
        fake_logits = self.model.netD(fake_data)
        g_loss = -fake_logits.mean()
        g_loss.backward()
        self.optimizers['G'].step()
        self.losses['gan_G'].append(g_loss.item())

    def _train_discriminator_once(self, real_data):
        self.model.weight_clipping()
        self.optimizers['D'].zero_grad()
        fake_data = self.model.netG(real_data.shape[0])
        fake_logits = self.model.netD(fake_data.detach_())
        real_logits = self.model.netD(real_data)
        w_distance = real_logits.mean() - fake_logits.mean()
        d_loss = -w_distance
        # self.losses['dis_grad'].append(self._cal_dis_grad(real_data, fake_data))
        # self.optimizers['D'].zero_grad()
        d_loss.backward()
        self.optimizers['D'].step()
        self.losses['gan_D'].append(d_loss.item())
        self.dis_outputs['real'] += real_logits.flatten().tolist()
        self.dis_outputs['fake'] += fake_logits.flatten().tolist()


import os
from dotenv import load_dotenv
import requests
import io
from PIL import Image
import streamlit as st
from streamlit_drawable_canvas import st_canvas
from utils import footer
import json
import base64

# Load environment variables
load_dotenv()

# Set up page config
st.set_page_config(
    page_title="Image Captioning",
    page_icon="ðŸŽ¨",
    layout="wide",
    initial_sidebar_state="expanded",
)

# Set up the title and a brief description
st.title("Image Captioning")
st.markdown("Capture the essence of your image with a caption!")

st.sidebar.markdown("""
## ðŸŽ‰ Welcome to Image Captioning! ðŸŽ‰

You know how they say a picture is worth a thousand words? Well, we're about to prove them wrong.

Here, we believe that a picture is worth exactly one caption. No more, no less.

So go ahead, upload an image or draw one if you're feeling artistic. Our state-of-the-art AI will then generate a caption that captures the essence of your image in a single sentence.

It's like magic, but without the rabbits and hats. Enjoy!
""")

# Get API token and URL from environment variables
API_TOKEN = os.getenv('API_TOKEN')
API_URL = os.getenv('IMAGE_CAPTIONING_API_URL')
headers = {"Authorization": f"Bearer {API_TOKEN}"}

# Function to send a POST request to the inference server
def query(payload):
    response = requests.post(API_URL, headers=headers, json=payload)
    return response.content

# Function to get caption from inference server
def get_caption_from_inference_server(image):
    image = image.convert("RGB")  # Convert image to RGB
    buffered = io.BytesIO()
    image.save(buffered, format="JPEG")
    img_str = base64.b64encode(buffered.getvalue())
    response = query({"inputs": img_str.decode('utf-8'), })
    response_json = json.loads(response.decode('utf-8'))  # Decode byte string and parse JSON
    caption = response_json[0]['generated_text']  # Extract generated text
    return caption

# Select image source
image_source = st.radio("Select image source", ["Upload image", "Draw image"])

if image_source == "Upload image":
    # Upload image
    uploaded_file = st.file_uploader("Upload an image", type=["png", "jpg", "jpeg"])
    if uploaded_file is not None:
        image = Image.open(uploaded_file)
        st.image(image, caption="Uploaded Image", width=400)
else:
    # Draw image
    st.markdown("Draw an image:")
    stroke_width = st.slider("Stroke width:", 1, 25, 10)
    stroke_color = st.color_picker("Stroke color:", '#e00')
    background_color = st.color_picker("Background color:", '#fff')

    canvas_result = st_canvas(
        fill_color="rgba(255, 165, 0, 0.3)",  # Fixed fill color with some opacity
        stroke_width=stroke_width,
        stroke_color=stroke_color,
        background_color=background_color,
        height=500,  # Adjust the height of the canvas
        width=1000,  # Adjust the width of the canvas
        drawing_mode="freedraw",
        key="canvas",
    )

    # Convert the drawn image to PIL Image
    if canvas_result.image_data is not None:
        drawn_image = Image.fromarray(canvas_result.image_data.astype('uint8'), 'RGBA')
        st.image(drawn_image, caption="Drawn Image", use_column_width=False)

# Button to generate caption
if st.button('Generate Caption'):
    with st.spinner('Generating your caption...'):
        if image_source == "Upload image" and uploaded_file is not None:
            caption = get_caption_from_inference_server(image)
        elif image_source == "Draw image" and canvas_result.image_data is not None:
            caption = get_caption_from_inference_server(drawn_image)
        else:
            st.warning("Please upload or draw an image first.")
            st.stop()
    st.success('Caption generated successfully!')
    st.markdown(f"
{caption}
", unsafe_allow_html=True)

# Display footer
st.markdown(footer,unsafe_allow_html=True)


import os
import io
import json
import tarfile
import glob
import time
import logging

import base64

import boto3

from datetime import datetime

lambda_client = boto3.client('lambda')

def lambda_handler(event, context):

    now = datetime.now()
    dt_string = now.strftime("%Y/%m/%d %H:%M:%S")

    print("Sending stuff to Lambda_2")
    event = json.loads(event['body'])
    inputParams = {
        'device_id': event['device_id'],
        'company_name': event['company_name'],
        # 'time': event['time'],
        'time': dt_string,
        'image': event['base64str']
    }

    response = lambda_client.invoke(
        FunctionName = 'arn:aws:lambda:us-east-2:882207224999:function:pi_camera_ppe_detection_v1_L_2',
        InvocationType = 'RequestResponse',
        Payload = json.dumps(inputParams)
    )

    response = lambda_client.invoke(
        FunctionName = 'arn:aws:lambda:us-east-2:882207224999:function:pi_camera_ppe_detection_v1_L_3',
        InvocationType = 'RequestResponse',
        Payload = json.dumps(inputParams)
    )

    return {
        "statusCode": 200,
        "body": json.dumps({'Status': "Successful"})
    }


#-*- coding: utf-8 -*-
#!/usr/bin/python3
import os
import sqlite3

patient = input('Name of the person you want to search: ' + str())

conn = sqlite3.connect('../db/pacientes.db')

try:
    c = conn.cursor()
    c.execute('SELECT id FROM pacientes WHERE nombre = ?', (str(patient),))
    namedb = str(c.fetchone()[0]) + '.db'
    conn.commit()
    conn.close()
    if os.path.isfile('../db/registros/'+namedb):
        conn = sqlite3.connect('../db/registros/'+namedb)

        c = conn.cursor()

        c.execute('SELECT * FROM datos')

        print(c.fetchall())

        conn.commit()
        conn.close()
    else:
        print('No records founded')

except TypeError:
    print('No records founded')


"""Returns bots ping, uptime and current load.
"""
from datetime import datetime
from math import ceil
from subprocess import check_output

def handle_update(bot, update, **kwargs):
    sent_time = update.message.date
    now = datetime.now()
    delta = now - sent_time

    ping = ceil(delta.total_seconds() * 1000)
    uptime = check_output(['uptime', '-p']).decode().split(' ', 1)[1]

    output = check_output(['cat', '/proc/loadavg']).split()[1].decode()
    load_5m = float(output) * 100

    report = ("Ping: {}ms\nBot uptime: {}"
              "Load over last 5 minutes: {}%").format(ping, uptime, load_5m)

    bot.sendMessage(update.message.chat_id, text=report)


import math, numpy
import matplotlib.pyplot as plt

# Free off-diagonal density matrix
def rho_free(x, xp, beta):
    return (math.exp(-(x - xp) ** 2 / (2.0 * beta)) /
            math.sqrt(2.0 * math.pi * beta))

def rho_harmonic_trotter(grid, beta):
    return numpy.array([[rho_free(x, xp, beta) * \
                         numpy.exp(-0.5 * beta * 0.5 * (x ** 2 + xp ** 2)) \
                         for x in grid] for xp in grid])

def V(x,cubic,quartic):
	return 0.5*x*x + cubic*x*x*x+ quartic*x*x*x*x
def rho_a_trotter(grid, beta,c,q):
    return numpy.array([[rho_free(x, xp, beta) * \
                         numpy.exp(-0.5 * beta *(V(x,c,q)+ V(xp,c,q))) \
                         for x in grid] for xp in grid])

def pi_quant(x,beta):
	a = math.sqrt(math.tanh(0.5*beta)/math.pi)
	return a*math.exp(-x*x*math.tanh(0.5*beta))
quartic = 1.
cubic = -quartic
x_max = 5.
nx = 100
dx = 2.0 * x_max / (nx - 1)
x = [i * dx for i in range(-(nx - 1) / 2, nx / 2 + 1)]
beta_tmp = 2.0 ** (-6)                   # initial value of beta (power of 2)
beta     = 2.0 ** 2                      # actual value of beta (power of 2)
rho = rho_a_trotter(x, beta_tmp,cubic,quartic)  # density matrix at initial beta
while beta_tmp < beta:
    rho = numpy.dot(rho, rho)
    rho *= dx
    beta_tmp *= 2.0

Z = sum(rho[j,j] for j in range(nx+1))*dx
pi_of_x = [ rho[j,j]/ Z for j in range(nx+1)]
f = open('data_aharm_matrixsquare_beta'+str(beta)+'.dat','w')
for j in range(nx+1):
	f.write(str(x[j])+' '+str(rho[j,j]/Z)+'\n')
f.close()

plt.scatter(x,pi_of_x,label="mat. sq.")
plt.title("beta="+str(beta)+"   cubic="+str(cubic)+"    quartic="+str(quartic))
plt.legend()
plt.show()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Bash models for use in scripts."""
from uuid import uuid4
from typing import Union, Optional, List

from pydantic import BaseModel, Field, validator, UUID4

DOCUMENTATION_INDICATION_STR = '#/'
USAGE_STRING = '# usage'

class BashDocumentationLine(BaseModel):
    """Abstraction for documentation in bash script."""
    module_name: str
    documentation: Union[str, List[str]]
    markers: bool = True

    def __str__(self) -> str:
        """Representation of Bash Documentation."""
        contents = DOCUMENTATION_INDICATION_STR + 'Usage:\n\t./' + \
            self.module_name + ' [OPTIONS]\n'
        if isinstance(self.logic, str):
            contents += self.logic
        elif isinstance(self.logic, list):
            for item in self.logic:
                contents += item + '\n'
        markers = ''
        end_markers = ''
        if markers is True:
            markers += ' {{{1'
            end_markers += '# 1}}}'
        return USAGE_STRING + markers + '\n' + DOCUMENTATION_INDICATION_STR +\
            contents + end_markers + '\n'

# pylint: disable=too-few-public-methods
class BashVariable(BaseModel):
    """Construct a BashVariable."""
    key: str
    value: str = ""

    def __str__(self) -> str:
        """Representation of BashVariable."""
        return f'{self.key}={self.value}'

class BashFunction(BaseModel):
    """Construct a Bash function."""
    name: str
    logic: Union[list, str]
    markers: bool = False

    def __str__(self) -> str:
        """Representation of BashFunction."""
        contents = ""
        if isinstance(self.logic, str):
            contents += self.logic
        elif isinstance(self.logic, list):
            for item in self.logic:
                contents += ('\t' + item + '\n')
        markers = ""
        end_markers = ""
        if self.markers is True:
            markers += ' {{{2'
            end_markers += ' # 2}}}'
        return self.name + '() {' + markers + '\n' + contents + '}' + \
            end_markers + '\n'

class BashDocumentation(BaseModel):
    """Provide a Bash documentation script strint."""

class BashEnvironment(BaseModel):
    """Populate the BashEnvironment class."""
    environment_variables: List[BashVariable]
    bash_functions: List[BashFunction]

    class Config:
        """Options to the BaseModel config class."""
        allow_arbitrary_types = True

if __name__ == "__main__":
    print(BashVariable(key='LOGLEVEL', value='INFO'))
    print(BashFunction(name='test_function',
                       logic='ps aux\n',
                       markers=True))


from unittest.mock import patch

import pytest
from airflow.exceptions import AirflowException

from catalog.tests.dags.providers.provider_api_scripts.resources.json_load import (
    make_resource_json_func,
)
from providers.provider_api_scripts.smithsonian import SmithsonianDataIngester

# Set up test class
ingester = SmithsonianDataIngester()
_get_resource_json = make_resource_json_func("smithsonian")

def test_get_hash_prefixes_with_len_one():
    with patch.object(ingester, "hash_prefix_length", 1):
        actual_prefix_list = list(ingester._get_hash_prefixes())
    expect_prefix_list = [
        "0",
        "1",
        "2",
        "3",
        "4",
        "5",
        "6",
        "7",
        "8",
        "9",
        "a",
        "b",
        "c",
        "d",
        "e",
        "f",
    ]
    assert actual_prefix_list == expect_prefix_list

def test_alert_new_unit_codes():
    sub_prov_dict = {"sub_prov1": {"a", "c"}, "sub_prov2": {"b"}, "sub_prov3": {"e"}}
    unit_code_set = {"a", "b", "c", "d"}
    with patch.dict(ingester.sub_providers, sub_prov_dict, clear=True):
        actual_codes = ingester._get_new_and_outdated_unit_codes(unit_code_set)
    expected_codes = ({"d"}, {"e"})
    assert actual_codes == expected_codes

@pytest.mark.parametrize(
    "new_unit_codes, outdated_unit_codes",
    [
        ({"d"}, {"e"}),
        ({"d"}, set()),
        (set(), {"e"}),
    ],
)
def test_validate_unit_codes_from_api_raises_exception(
    new_unit_codes, outdated_unit_codes
):
    with patch.object(ingester, "_get_unit_codes_from_api"), patch.object(
        ingester,
        "_get_new_and_outdated_unit_codes",
        return_value=(new_unit_codes, outdated_unit_codes),
    ):
        message = "^\n\\*Updates needed to the SMITHSONIAN_SUB_PROVIDERS dictionary\\**"
        with pytest.raises(AirflowException, match=message):
            ingester.validate_unit_codes_from_api()

def test_validate_unit_codes_from_api():
    with patch.object(ingester, "_get_unit_codes_from_api"), patch.object(
        ingester, "_get_new_and_outdated_unit_codes", return_value=(set(), set())
    ):
        # Validation should run without raising an exception
        ingester.validate_unit_codes_from_api()

@pytest.mark.parametrize(
    "input_int, expect_len, expect_first, expect_last",
    [
        (1, 16, "0", "f"),
        (2, 256, "00", "ff"),
        (3, 4096, "000", "fff"),
        (4, 65536, "0000", "ffff"),
    ],
)
def test_get_hash_prefixes_with_other_len(
    input_int, expect_len, expect_first, expect_last
):
    with patch.object(ingester, "hash_prefix_length", input_int):
        actual_list = list(ingester._get_hash_prefixes())

    assert all("0x" not in h for h in actual_list)
    assert all(
        int(actual_list[i + 1], 16) - int(actual_list[i], 16) == 1
        for i in range(len(actual_list) - 1)
    )
    assert len(actual_list) == expect_len
    assert actual_list[0] == expect_first
    assert actual_list[-1] == expect_last

def test_get_next_query_params_first_call():
    hash_prefix = "ff"
    actual_params = ingester.get_next_query_params(
        prev_query_params=None, hash_prefix=hash_prefix
    )
    actual_params.pop("api_key")  # Omitting the API key
    expected_params = {
        "q": f"online_media_type:Images AND media_usage:CC0 AND hash:{hash_prefix}*",
        "start": 0,
        "rows": 1000,
    }
    assert actual_params == expected_params

def test_get_next_query_params_updates_parameters():
    previous_query_params = {
        "api_key": "pass123",
        "q": "online_media_type:Images AND media_usage:CC0 AND hash:ff*",
        "start": 0,
        "rows": 1000,
    }
    actual_params = ingester.get_next_query_params(previous_query_params)
    expected_params = {
        "api_key": "pass123",
        "q": "online_media_type:Images AND media_usage:CC0 AND hash:ff*",
        "start": 1000,
        "rows": 1000,
    }
    assert actual_params == expected_params

@pytest.mark.parametrize(
    "input_row, field, expected_dict",
    [
        # field: descriptiveNonRepeating
        ({}, "descriptiveNonRepeating", {}),
        ({"content": {"key": {"key2", "val2"}}}, "descriptiveNonRepeating", {}),
        (
            {"noncontent": {"descriptiveNonRepeating": {"key2": "val2"}}},
            "descriptiveNonRepeating",
            {},
        ),
        (
            {"content": {"descriptiveNonRepeating": {"key2": "val2"}}},
            "descriptiveNonRepeating",
            {"key2": "val2"},
        ),
        # field: indexedStructured
        ({}, "indexedStructured", {}),
        ({"content": {"key": {"key2", "val2"}}}, "indexedStructured", {}),
        (
            {"noncontent": {"indexedStructured": {"key2": "val2"}}},
            "indexedStructured",
            {},
        ),
        (
            {"content": {"indexedStructured": {"key2": "val2"}}},
            "indexedStructured",
            {"key2": "val2"},
        ),
        # field: freetext
        ({}, "freetext", {}),
        ({"content": {"key": {"key2", "val2"}}}, "freetext", {}),
        ({"noncontent": {"freetext": {"key2": "val2"}}}, "freetext", {}),
        ({"content": {"freetext": {"key2": "val2"}}}, "freetext", {"key2": "val2"}),
    ],
)
def test_content_dict(input_row, field, expected_dict):
    actual_dict = ingester._get_content_dict(input_row, field)
    assert actual_dict == expected_dict

@pytest.mark.parametrize(
    "required_type,default_input",
    [
        (str, ""),
        (int, 0),
        (float, 0.0),
        (complex, 0j),
        (list, []),
        (tuple, ()),
        (dict, {}),
        (set, set()),
        (bool, False),
        (bytes, b""),
    ],
)
def test_check_type_with_defaults(required_type, default_input):
    assert ingester._check_type(default_input, required_type) == default_input

@pytest.mark.parametrize(
    "required_type,good_input",
    [
        (str, "abc"),
        (int, 5),
        (float, 1.2),
        (complex, 3 + 2j),
        (list, ["a", 2, 0, False]),
        (tuple, (1, 0, False)),
        (dict, {"key": "val", "f": False}),
        (set, {1, False, "abc"}),
        (bool, True),
        (bytes, b"abc"),
    ],
)
def test_check_type_with_truthy_good_inputs(required_type, good_input):
    assert ingester._check_type(good_input, required_type) == good_input

@pytest.mark.parametrize(
    "required_type,good_indices,default",
    [
        (str, (0, 1), ""),
        (int, (2, 3), 0),
        (float, (4, 5), 0.0),
        (complex, (6, 7), 0j),
        (list, (8, 9), []),
        (tuple, (10, 11), ()),
        (dict, (12, 13), {}),
        (set, (14, 15), set()),
        (bool, (16, 17), False),
        (bytes, (18, 19), b""),
    ],
)
def test_check_type_with_bad_inputs(required_type, good_indices, default):
    bad_input_list = [
        "abc",
        "",
        5,
        0,
        1.2,
        0.0,
        3 + 2j,
        0j,
        ["a", 2],
        [],
        (1, 2),
        (),
        {"key": "val"},
        {},
        {1, "abc"},
        set(),
        True,
        False,
        b"abc",
        b"",
    ]
    for i in sorted(good_indices, reverse=True):
        del bad_input_list[i]
    for bad_input in bad_input_list:
        assert ingester._check_type(bad_input, required_type) == default

@pytest.mark.parametrize(
    "input_dnr, expect_image_list",
    [
        ({}, []),
        ({"non_media": {"media": ["image1", "image2"]}}, []),
        ({"online_media": "wrong type"}, []),
        ({"online_media": ["wrong", "type"]}, []),
        ({"online_media": {"media": "wrong type"}}, []),
        ({"online_media": {"media": {"wrong": "type"}}}, []),
        (
            {
                "record_ID": "siris_arc_291918",
                "online_media": {"mediaCount": 1, "media": ["image1", "image2"]},
            },
            ["image1", "image2"],
        ),
    ],
)
def test_get_image_list(input_dnr, expect_image_list):
    input_row = {"key": "val"}
    with patch.object(
        ingester, "_get_content_dict", return_value=input_dnr
    ) as mock_dnr:
        actual_image_list = ingester._get_image_list(input_row)
    mock_dnr.assert_called_once_with(input_row, "descriptiveNonRepeating")
    assert actual_image_list == expect_image_list

def test_get_media_type():
    actual_type = ingester.get_media_type(record={})
    assert actual_type == "image"

@pytest.mark.parametrize(
    "input_dnr, expect_foreign_landing_url",
    [
        ({}, None),
        (
            {
                "guid": "http://fallback.com",
                "unit_code": "NMNHMAMMALS",
                "record_link": "http://chooseme.com",
            },
            "http://chooseme.com",
        ),
        ({"guid": "http://fallback.com"}, "http://fallback.com"),
        ({"no": "urlhere"}, None),
    ],
)
def test_get_foreign_landing_url(input_dnr, expect_foreign_landing_url):
    input_row = {"key": "val"}
    with patch.object(
        ingester, "_get_content_dict", return_value=input_dnr
    ) as mock_dnr:
        actual_foreign_landing_url = ingester._get_foreign_landing_url(input_row)
    mock_dnr.assert_called_once_with(input_row, "descriptiveNonRepeating")
    assert actual_foreign_landing_url == expect_foreign_landing_url

@pytest.mark.parametrize(
    "input_is, input_ft, expected_creator",
    [
        ({}, {}, None),
        ({"name": ["Alice"]}, {}, None),
        ({"name": "alice"}, {}, None),
        ({"name": [{"type": ["personal", "main"], "nocontent": "Alice"}]}, {}, None),
        ({"name": [{"type": "personal_main", "nocontent": "Alice"}]}, {}, None),
        ({"noname": [{"type": "personal_main", "content": "Alice"}]}, {}, None),
        ({"name": [{"label": "personal_main", "content": "Alice"}]}, {}, None),
        ({"name": [{"type": "impersonal_main", "content": "Alice"}]}, {}, None),
        (
            {"name": [{"type": "personal_main", "content": "Alice"}]},
            {"name": "Bob"},
            "Alice",
        ),
        (
            {"name": [{"type": "personal_main", "content": "Alice"}]},
            {"name": ["Bob"]},
            "Alice",
        ),
        (
            {"name": [{"type": "personal_main", "content": "Alice"}]},
            {"name": [{"label": "Creator", "nocontent": "Bob"}]},
            "Alice",
        ),
        (
            {"name": [{"type": "personal_main", "content": "Alice"}]},
            {"name": [{"nolabel": "Creator", "content": "Bob"}]},
            "Alice",
        ),
        (
            {"name": [{"type": "personal_main", "content": "Alice"}]},
            {"name": [{"label": "NotaCreator", "content": "Bob"}]},
            "Alice",
        ),
        (
            {"name": [{"type": "personal_main", "content": "Alice"}]},
            {"noname": [{"label": "Creator", "content": "Bob"}]},
            "Alice",
        ),
        (
            {"name": [{"type": "personal_main", "content": "Alice"}]},
            {"name": [{"label": "Creator", "content": "Bob"}]},
            "Bob",
        ),
        (
            {},
            {
                "name": [
                    {"label": "Designer", "content": "Alice"},
                    {"label": "Creator", "content": "Bob"},
                ]
            },
            "Bob",
        ),
        (
            {},
            {
                "name": [
                    {"label": "AFTER", "content": "Bob"},
                    {"label": "Designer", "content": "Alice"},
                ]
            },
            "Alice",
        ),
        (
            {},
            {
                "name": [
                    {"label": "AFTER", "content": "Bob"},
                    {"label": "DESIGNER", "content": "Alice"},
                ]
            },
            "Alice",
        ),
        (
            {
                "name": [
                    {"type": "personal_main", "content": "Alice"},
                    {"type": "corporate_subj", "content": "Zoological Park"},
                ]
            },
            {
                "name": [
                    {"label": "Creator", "content": "Bob"},
                    {"label": "Subject", "content": "Zoological Park"},
                ]
            },
            "Bob",
        ),
    ],
)
def test_get_creator(input_is, input_ft, expected_creator):
    creator_types = {"creator": 0, "designer": 1, "after": 3}
    input_row = {
        "test": "row",
        "content": {"freetext": input_ft, "indexedStructured": input_is},
    }
    with patch.object(ingester, "creator_types", creator_types):
        actual_creator = ingester._get_creator(input_row)

    assert actual_creator == expected_creator

@pytest.mark.parametrize(
    "input_ft, input_dnr, expected_description",
    [
        ({}, {}, None),
        ({"notes": [{"label": "notthis", "content": "blah"}]}, {}, None),
        ({"notes": "notalist"}, {}, None),
        ({"notes": [{"label": "Summary", "content": "blah"}]}, {}, "blah"),
        (
            {
                "notes": [
                    {"label": "Description", "content": "blah"},
                    {"label": "Summary", "content": "blah"},
                    {"label": "Description", "content": "blah"},
                ]
            },
            {},
            "blah blah blah",
        ),
        (
            {
                "notes": [
                    {"label": "notDescription", "content": "blah"},
                    {"label": "Summary", "content": "blah"},
                    {"label": "Description", "content": "blah"},
                ]
            },
            {},
            "blah blah",
        ),
    ],
)
def test_ext_meta_data_description(input_ft, input_dnr, expected_description):
    description_types = {"description", "summary"}
    input_row = {
        "test": "row",
        "content": {"freetext": input_ft, "descriptiveNonRepeating": input_dnr},
    }
    with patch.object(ingester, "description_types", description_types):
        meta_data = ingester._extract_meta_data(input_row)
    actual_description = meta_data.get("description")

    assert actual_description == expected_description

@pytest.mark.parametrize(
    "input_ft, input_dnr, expected_label_text",
    [
        ({}, {}, None),
        ({"notes": [{"label": "notthis", "content": "blah"}]}, {}, None),
        ({"notes": "notalist"}, {}, None),
        ({"notes": [{"label": "Label Text", "content": "blah"}]}, {}, "blah"),
        (
            {
                "notes": [
                    {"label": "Label Text", "content": "blah"},
                    {"label": "Summary", "content": "halb"},
                    {"label": "Description", "content": "halb"},
                ]
            },
            {},
            "blah",
        ),
    ],
)
def test_ext_meta_data_label_text(input_ft, input_dnr, expected_label_text):
    input_row = {
        "test": "row",
        "content": {"freetext": input_ft, "descriptiveNonRepeating": input_dnr},
    }
    meta_data = ingester._extract_meta_data(input_row)
    actual_label_text = meta_data.get("label_text")

    assert actual_label_text == expected_label_text

@pytest.mark.parametrize(
    "input_ft, input_dnr, expected_meta_data",
    [
        ({"nothing": "here"}, {"nothing_to": "see"}, {}),
        ({}, {"unit_code": "SIA"}, {"unit_code": "SIA"}),
        (
            {},
            {"data_source": "Smithsonian Institution Archives"},
            {"data_source": "Smithsonian Institution Archives"},
        ),
    ],
)
def test_extract_meta_data_dnr_fields(input_ft, input_dnr, expected_meta_data):
    input_row = {
        "test": "row",
        "content": {"freetext": input_ft, "descriptiveNonRepeating": input_dnr},
    }
    actual_meta_data = ingester._extract_meta_data(input_row)

    assert actual_meta_data == expected_meta_data

@pytest.mark.parametrize(
    "input_is, expect_tags",
    [
        ({}, []),
        ({"nothing": "here"}, []),
        (
            {
                "date": ["", ""],
                "place": ["Indian Ocean"],
            },
            ["Indian Ocean"],
        ),
        (
            {
                "date": ["2000s"],
                "object_type": ["Holotypes", "Taxonomic type specimens"],
                "topic": ["Paleogeneral", "Protists"],
                "place": ["Indian Ocean"],
            },
            [
                "2000s",
                "Holotypes",
                "Taxonomic type specimens",
                "Paleogeneral",
                "Protists",
                "Indian Ocean",
            ],
        ),
    ],
)
def test_extract_tags(input_is, expect_tags):
    input_row = {"test": "row"}
    get_is = patch.object(ingester, "_get_content_dict", return_value=input_is)
    with get_is as mock_is:
        actual_tags = ingester._extract_tags(input_row)
    mock_is.assert_called_once_with(input_row, "indexedStructured")
    assert actual_tags == expect_tags

@pytest.mark.parametrize(
    "input_media, expected_image_data",
    [
        ([], []),
        (
            [
                {
                    "thumbnail": "https://thumbnail.one",
                    "idsId": "id_one",
                    "usage": {"access": "CC0"},
                    "guid": "http://gu.id.one",
                    "type": "Images",
                    "content": "",
                },
            ],
            [],
        ),
        (
            [
                {
                    "thumbnail": "https://thumbnail.one",
                    "idsId": "",
                    "usage": {"access": "CC0"},
                    "guid": "http://gu.id.one",
                    "type": "Images",
                    "content": "https://image.url.one",
                }
            ],
            [],
        ),
        (
            [
                {
                    "thumbnail": "https://thumbnail.one",
                    "idsId": "id_one",
                    "usage": {"access": "CC0"},
                    "guid": "http://gu.id.one",
                    "type": "Images",
                    "content": "https://image.url.one",
                },
                {
                    "thumbnail": "https://thumbnail.two",
                    "idsId": "id_two",
                    "usage": {"access": "CC0"},
                    "guid": "http://gu.id.two",
                    "type": "Images",
                    "content": "https://image.url.two",
                },
            ],
            [
                {
                    "foreign_identifier": "id_one",
                    "url": "https://image.url.one",
                },
                {
                    "foreign_identifier": "id_two",
                    "url": "https://image.url.two",
                },
            ],
        ),
    ],
)
def test_process_image_list(input_media, expected_image_data):
    shared_image_data = {
        "foreign_landing_url": "https://foreignlanding.url",
        "title": "The Title",
        "license_info": ingester.license_info,
        "creator": "Alice",
        "meta_data": {"unit_code": "NMNHBOTANY"},
        "source": "smithsonian_national_museum_of_natural_history",
        "raw_tags": ["tag", "list"],
    }
    expected_result = [(img | shared_image_data) for img in expected_image_data]
    actual_image_data = ingester._get_associated_images(input_media, shared_image_data)
    assert actual_image_data == expected_result

def test_get_record_data():
    data = _get_resource_json("actual_record_data.json")
    actual_data = ingester.get_record_data(data)
    expected_data = [
        {
            "url": "https://collections.nmnh.si.edu/media/?irn=15814382",
            "foreign_identifier": "https://collections.nmnh.si.edu/media/?irn=15814382",
            "foreign_landing_url": "http://n2t.net/ark:/65665/34857ca78-9195-4156-849b-1ec47f7cd1ce",
            "title": "Passerculus sandwichensis nevadensis",
            "license_info": ingester.license_info,
            "source": "smithsonian_national_museum_of_natural_history",
            "creator": "Seymour H. Levy",
            "meta_data": {
                "unit_code": "NMNHBIRDS",
                "data_source": "NMNH - Vertebrate Zoology - Birds Division",
            },
            "raw_tags": [
                "1950s",
                "Animals",
                "Birds",
                "United States",
                "Pinal",
                "North America",
                "Arizona",
            ],
        }
    ]

    assert actual_data == expected_data


import json
from typing import Optional

from sommelier.adapters.sockets import (
    WebSocketReader, WebSocketWriter
)
from sommelier.behave_wrapper import FlowListener, response_json_holder
from sommelier.behave_wrapper.tables import carpenter
from sommelier.utils import require_var

class WSocketManager(FlowListener):
    """
    This is a one way web socket manager
    Manager allows to do read, write or both but via 2 different ws
    Tests written in behave are sequential and WS manager fits into this idea
    """

    def __init__(self, host):
        super().__init__()
        require_var(host, "host")
        self.host = host
        self.reader: Optional[WebSocketReader] = None
        self.writer: Optional[WebSocketWriter] = None

    def after_all(self):
        super().after_all()
        self.disconnect_all()

    def create_socket(self, reader):
        cookies = carpenter.builder().double().dict()
        if reader:
            self.reader = WebSocketReader(self.host, cookies)
            self.reader.schedule_start()
        else:
            self.writer = WebSocketWriter(self.host, cookies)
            self.writer.schedule_start()

    def read_data(self):
        response_json_holder.with_description("WS-Read", self.host)
        text = self.reader.read_one()
        data = None
        if text is not None:
            data = json.loads(text)
        response_json_holder.hold(data)

    def write_data(self):
        data = carpenter.builder().double().dict()
        self.writer.write(data)

    def disconnect_all(self):
        if self.reader is not None:
            self.reader.schedule_shutdown()
        if self.writer is not None:
            self.writer.schedule_shutdown()


from dataclasses import asdict, dataclass
from enum import Enum
from typing import Callable, Sequence

import numpy as np
import pandas as pd
from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score

from .ground_truths import SplitGroundTruths, TrainValTestGroundTruths
from .models import ModelNames
from .predictions import SplitPredictions, TrainValTestPredictions

@dataclass
class SplitScores:
    bust: float
    hip: float
    waist: float

@dataclass
class TrainValTestScores:
    train: SplitScores
    val: SplitScores
    test: SplitScores

@dataclass
class Metrics:
    mae: TrainValTestScores
    mse: TrainValTestScores
    r2: TrainValTestScores

class MetricNames(Enum):
    mae = "mae"
    mse = "mse"
    r2 = "r2"

class MetricsManager:
    def __init__(self) -> None:
        self.metrics: list[Metrics] = []
        self.model_names: list[ModelNames] = []
        self.metrics_table = pd.DataFrame([])

    def update_metrics(
        self,
        ground_truths: TrainValTestGroundTruths,
        predictions: TrainValTestPredictions,
    ) -> pd.DataFrame:
        mae = compute_train_val_test_metrics(
            mean_absolute_error, ground_truths, predictions
        )
        mse = compute_train_val_test_metrics(
            mean_squared_error, ground_truths, predictions
        )
        r2 = compute_train_val_test_metrics(r2_score, ground_truths, predictions)
        self.metrics.append(Metrics(mae, mse, r2))
        self.model_names.append(predictions.model_names)

        all_metric_dfs: list[pd.DataFrame] = []

        for metric in MetricNames:
            train_test_val_scores: TrainValTestScores = getattr(
                self.metrics[-1], metric.value
            )
            metric_df = self.get_single_metric_df(train_test_val_scores, metric)
            all_metric_dfs.append(metric_df)

        table_update = pd.concat(all_metric_dfs, axis=1)
        self.metrics_table = pd.concat([self.metrics_table, table_update])
        return self.metrics_table

    def get_single_metric_df(
        self, train_test_val_scores: TrainValTestScores, metric_name: MetricNames
    ) -> pd.DataFrame:
        metric_df = pd.DataFrame(asdict(train_test_val_scores))
        metric_df.columns = [col + f"_{metric_name.value}" for col in metric_df.columns]

        model_names = self.model_names[-1]
        if list(metric_df.index) == ["bust", "hip", "waist"]:
            metric_df.index = [model_names.bust, model_names.hip, model_names.waist]

        return metric_df

    def show_validation_metrics(self) -> pd.DataFrame:
        validation_columns = [
            col for col in self.metrics_table.columns if col.startswith("val")
        ]
        return self.metrics_table.loc[:, validation_columns]

def compute_train_val_test_metrics(
    score_func: Callable[[Sequence, np.ndarray], float],
    gound_truths: TrainValTestGroundTruths,
    predictions: TrainValTestPredictions,
) -> TrainValTestScores:
    train = compute_split_metrics(score_func, gound_truths.train, predictions.train)
    val = compute_split_metrics(score_func, gound_truths.val, predictions.val)
    test = compute_split_metrics(score_func, gound_truths.test, predictions.test)
    return TrainValTestScores(train, val, test)

def compute_split_metrics(
    score_func: Callable[[Sequence, np.ndarray], float],
    split_ground_truths: SplitGroundTruths,
    split_predictions: SplitPredictions,
) -> SplitScores:
    bust = score_func(split_ground_truths.bust, split_predictions.bust)
    hip = score_func(split_ground_truths.hip, split_predictions.hip)
    waist = score_func(split_ground_truths.waist, split_predictions.waist)

    return SplitScores(bust, hip, waist)


#!/usr/bin/env python3
"""
Convert Southern Poverty Law Center Cryptocurrency Report to a TagPack.
"""

import os
import csv
from datetime import datetime, date
from typing import List

import yaml

CURRENCY = {
    'Bitcoin Addresses': 'BTC',
    'Ethereum Addresses': 'ETH',
    'Litecoin Addresses': 'LTC',
    'Monero Address': 'XMR'
}

class RawData:
    """
    Download and read data provided by the source.
    """
    def __init__(self, fn: str, url: str):
        self.fn = fn
        self.url = url

    def download(self):
        from urllib.request import urlretrieve
        urlretrieve(self.url, self.fn)

    def read(self) -> List[dict]:
        with open(self.fn, 'r',encoding='utf-8', newline='') as csvfile:
            reader = csv.DictReader(csvfile, delimiter=',', quotechar='"')
            return [row for row in reader]

class TagPackGenerator:
    """
    Generate a TagPack from SPLC Cryptocurrency Report.
    """

    def __init__(self, rows: List[dict], title: str, creator: str, description: str, lastmod: date, source: str):
        self.rows = rows
        self.data = {
            'title': title,
            'creator': creator,
            'description': description,
            'lastmod': lastmod,
            'abuse': 'extremism',
            'category': 'user',  # like in the OFAC TagPack generator
            'source': source,
            'tags': []
        }
        self.source = source

    def generate(self):
        tags = []
        for row in self.rows:
            for column in ['Bitcoin Addresses', 'Ethereum Addresses', 'Litecoin Addresses', 'Monero Address']:
                for address in row[column].split('\n'):
                    if not address or address == 'uses':
                        continue
                    tag = {
                        'address': address.strip(),
                        'currency': CURRENCY[column],
                        'label': row['Entity']
                    }
                    tags.append(tag)
        self.data['tags'] = tags

    def saveYaml(self, fn: str):
        with open(fn, 'w', encoding='utf-8') as f:
            f.write(yaml.dump(self.data, sort_keys=False))

if __name__ == '__main__':
    with open('config.yaml', 'r') as config_file:
        config = yaml.safe_load(config_file)

    raw_data = RawData(config['RAW_FILE_NAME'], config['URL'])
    if not os.path.exists(config['RAW_FILE_NAME']):
        raw_data.download()

    last_mod = datetime.fromtimestamp(os.path.getmtime(config['RAW_FILE_NAME'])).date()
    generator = TagPackGenerator(raw_data.read(), config['TITLE'], config['CREATOR'], config['DESCRIPTION'],
                                 last_mod, config['SOURCE'])
    generator.generate()
    generator.saveYaml(config['TAGPACK_FILE_NAME'])


import logging
from pprint import pformat, pprint
from flask_jsonrpc.proxy import ServiceProxy

logger = logging.getLogger(__name__)

AZURE = "http://ftr-app.japanwest.cloudapp.azure.com:5000"
LOCAL = "http://localhost:5000"

SERVER_URL = AZURE
API_URI = '/api'

server = ServiceProxy(SERVER_URL + API_URI)

def call_debug(func,*args,**kwargs):
	def wrap():
		print("")
		print("-"*80)
		print(" METHOD - {}".format(func.__name__))
		print(" API_SERVER - {}".format(SERVER_URL))
		print("-"*80)
		r = func(*args,**kwargs)
		print("[ result ]")
		print(pformat(r))
		print("")
	return wrap

def rpc_call(func,*args,**kwargs):
    try:
        logger.info("--CALL")
        r = func(*args,**kwargs)
        logger.info(r)
        if 'error' in r:
            return r.get('error').get('name')
        else:
            return r.get('result')
    except Exception as e:
        logger.error(e)
        return None


from captcha.image import ImageCaptcha
import random

import captcha

if __name__ == "__main__":
    # Create an image instance of the given size
    sizes = []
    for i in range(1, 10):
        sizes.append(random.randint(110, 170))

    imageCaptcha = ImageCaptcha(fonts=["./storage/fonts/captchaFont.ttf"], font_sizes=sizes, width=500, height=300)
    # ImageCaptcha.create_noise_dots(image, color = (0, 0, 0), width = 1, number = 100)

    possibleLetters = "ABCEFGHJKLMNOPRSTUVWXYZ"

    captchaStr = " "
    for i in range(0, 5):
        captchaStr += random.choice(possibleLetters)
        captchaStr += " " * random.randint(1, 4)

    # generate the image of the given text
    data = imageCaptcha.generate(captchaStr)
    # image.create_noise_dots(image, color = (0, 0, 0), width = 1, number = 100)
    print(captchaStr)

    # write the image on the given file and save it
    imageCaptcha.write(captchaStr, "CAPTCHA.png")


import sys
import json
import requests
import re

# Function for HTTPRequestAction call
def requestAction(action, data):
    url = action['options']['url']
    # if url contains interpolated values, use function
    if "{{" and "}}" in url:
        url = interpolation(url, storedData)
    r = requests.get(url)
    # if 2xx HTTP response gather request data
    if r.status_code == 200 or 204:
        storedData2 = r.text
        parse_json = json.loads(storedData2)
        return parse_json
    # implement Network failure and termination for non-2xx HTTP response
    else:
        print('Network Failure. Terminating Program')
        sys.exit()

# Function to interpolate values to string/url
def interpolation(string, inputData):
    # identify and store values between dounble curly braces
    text_in_brackets = re.findall('[^{\{]+(?=}\})', string)
    keys = []
    values = []
    # loop to split values
    for i in text_in_brackets:
        keys.append(i.split("."))
    # double loop to identify and store keys values from HTTPRequest data
    for j in range(0, len(keys)):
        temp = storedData
        for k in range(0, len(keys[j])):
            # try except to store found data and store blank if key not valued
            try:
                temp = temp[keys[j][k]]
            except:
                temp = ''
        values.append(temp)
    # iterate through string and replace curly braces keys with stored values
    for l in range(0, len(text_in_brackets)):
        string = string.replace(
            "{{"+str(text_in_brackets[l])+"}}", str(values[l]))
    return string

# locate json file from command line argument
filename = "datafiles/" + sys.argv[1]
try:
    with open(filename) as json_file:
        # gather data from data file and convert to json format
        actionsObject = json.load(json_file)
        # object to store all HTTPRequestAction data
        storedData = {}
        # iterate through actions to store & trigger function depending on type
        for action in actionsObject['actions']:
            if action['type'] == "HTTPRequestAction":
                # Request data and store with actions name as value
                storedData[action['name']] = requestAction(action, storedData)
            if action['type'] == "PrintAction":
                # Gather string, interpolate and print to console
                printString = action['options']['message']
                message = interpolation(printString, storedData)
                print(message)
# implement Network failure and termination if file not located
except FileNotFoundError:
    print('Invalid file name.')
    sys.exit()


# -*- coding: utf-8 -*-
import scrapy
import re
from MoguSpider.items import SuningSpiderItem
import datetime

class SuningSpider(scrapy.Spider):
    name = 'suning'
    allowed_domains = ['list.suning.com']
    start_urls = ['https://list.suning.com/']

    def parse(self, response):
        search_divs = response.css(".search-main > div")
        for search_div in search_divs:
            firstId = search_div.css("div::attr(id)").extract_first("")
            firstName = search_div.css("h2::text").extract_first("")
            suningItem = SuningSpiderItem()
            suningItem["Name"] = firstName
            suningItem["SourceId"] = firstId
            suningItem["ParentId"] = "0"
            suningItem['CreatedDate'] = datetime.datetime.now()
            suningItem['SyncTime'] = datetime.datetime.now()
            yield suningItem
            boxs = search_div.css(".title-box")
            for box in boxs:
                secondUrl = box.css(".t-left a::attr(href)").extract_first("")
                martch_re = re.match(".*-(\d+)-.*",secondUrl);
                secondId = martch_re.group(1)
                secondName = box.css(".t-left a::attr(title)").extract_first("")
                rights = box.css(".t-right a")
                suningItem["Name"] = secondName
                suningItem["SourceId"] = secondId
                suningItem["ParentId"] = firstId
                suningItem['CreatedDate'] = datetime.datetime.now()
                suningItem['SyncTime'] = datetime.datetime.now()
                yield suningItem
                for right in rights:
                    thirdUrl = right.css("a::attr(href)").extract_first("")
                    martch_re = re.match(".*-(\d+)-.*",thirdUrl);
                    thirdId = martch_re.group(1)
                    thirdName = right.css("a::text").extract_first("")
                    suningItem["Name"] = thirdName
                    suningItem["SourceId"] = thirdId
                    suningItem["ParentId"] = secondId
                    suningItem['CreatedDate'] = datetime.datetime.now()
                    suningItem['SyncTime'] = datetime.datetime.now()
                    yield suningItem
        pass


import streamlit as st
import pandas as pd
import datetime
from PIL import Image
import matplotlib.pyplot as plt
st.title("Hello World")
st.text('Bu bir text')
st.markdown('# This is a markdown text')
st.markdown('`this is a markdown text`')
st.header('this is a header')
st.subheader('this is subheader')
st.success('Congrats')
st.info('Warning!!!')
st.error('Error')
st.help(map)
st.write('this is a text written with write function')
img = Image.open('cat.jpg')
st.image(img,caption = 'Cat',use_column_width = True)
st.video('https://www.youtube.com/watch?v=eX2qFMC8cFo')
#checkbox
if st.checkbox('hide and seek'):
    st.write('Seek')

status = st.radio('What is your favorite color?',('red','green','blue'))
st.write(f'your favorite color is {status}')
button = st.button('press me!!!')
if button:
    st.success('button just clicked')

number = st.selectbox('select a numver',(1,2,3,4,5))
st.write(f'you choose {number}')

st.multiselect('select multiple numbers',[1,2,3])

st.slider('Select a number',0,9,3,2) # 3 default, 2 stepsize

name = st.text_input('Enter Your Name')
st.write(f'your name is {name}')

if st.button('Submit'):
    st.write(f'Hello {name.swapcase()} ')

st.text_area('Enter a message','let go')

st.date_input('date')

st.time_input('time')

st.code('import pandas as pd')

with st.echo():
    import numpy as np
    import pandas as pd
    import matplotlib.pyplot as plt
    df = pd.DataFrame({'a': np.random.randint(1,10,5),
                      'b':np.arange(0,10,2)})
    df

st.sidebar.title('Side Bar')
st.sidebar.text_area('text in side bar')
st.sidebar.slider('input',0,5,1,1)

df = pd.read_csv('Mall_Customers.csv')
st.table(df.head(10))
st.sidebar.table(df.head().iloc[:3,:3])
st.write(df.head())  # dinamik

st.dataframe(df.head())

st.line_chart(df.Age)

slider = st.slider('x')
seri = np.random.randn(slider)
st.line_chart(seri)
st.write(seri.plot())


import redis

from DBHelper import DBHelper

"""
1.查找数据，首先到redis查找
   2.如果找到，则直接使用
   3. 如果找不到，去mysql中查找
       4 在mysql找到了，首先将数据添加redis中，然后再使用
       5 如果没找到，报错

"""
red = redis.StrictRedis()
db = DBHelper('student')

# 1 在redis中查找
sno = input("请输入学号：")
# 键就是学号
student = red.hgetall(sno)

if student:
    # 2 打印
    print("redis:")
    print(student)
else:
    #     3 到mysql中查找
    data = db.where(sid=sno).select()
    print("mysql:",data)
    # 4 将数据添加到reids
    if data:

        red.hmset(sno,data[0])
    else:
        print("mysql没有信息")


import collections
import mlpy
import numpy as np
import scipy as sp
import csv
import string
from sklearn import linear_model

recipesPath = '../mealplan/recipeData.txt'

cuisineMap = {}
cuisineMap['NorthernEuropean'] =250
cuisineMap['WesternEuropean'] =2659
cuisineMap['SouthernEuropean'] =4180
cuisineMap['MiddleEastern'] =645
cuisineMap['SouthAsian'] =621
cuisineMap['LatinAmerican'] =2917
cuisineMap['NorthAmerican'] =41524
cuisineMap['EasternEuropean'] =381
cuisineMap['SoutheastAsian'] =457
cuisineMap['African'] =352
cuisineMap['EastAsian'] =2512

oppMap = {}
for k,v in cuisineMap.iteritems():
	oppMap[v] = k

def process(filename):
	ingredientSet = getIngreds()
	X = []
	Y = []
	ingredientsList = []
	countCuisine = 0
	with open(filename, 'rb') as dataset:
		datareader = csv.reader(dataset)
		tempX = []
		for row in datareader:
			cuisine = row[0]
			orig = row[1:]
			ingredients = collections.Counter(orig)
			ingredientsList.append(orig)
			tempX.append((ingredients, cuisineMap.get(cuisine)))
		for ingredients, cuisine in tempX:
			featureVector = [0] * len(ingredientSet)
			for index, ingredient in enumerate(ingredientSet):
				if ingredient in ingredients:
					featureVector[index] = 1
				else:
					featureVector[index] = 0
			X.append(featureVector)
			Y.append(cuisine)
		return X, Y, ingredientSet, cuisineMap, ingredientsList

def readIngredients(predictor, ingredientSet):
	ingredientsList = collections.OrderedDict()
	with open(recipesPath, 'rb') as dataset, open('./recipeWithCuisines.txt', 'wb') as output:
		for line in dataset:
			orig = line.strip()
			ingredients = []
			line = line.split("<>")
			ingredString = line[6]
			ingredList = ingredString.split(',')
			for ingred in ingredList:
				ingred = ingred.split(';')
				ingredients.append(ingred[0])
			testX = constructFeatureVector(ingredients, ingredientSet)
			cuisine = predictor.predict([testX])
			output.write(orig + "<>" + oppMap[cuisine[0]] + '\n')
	return ingredientsList

def constructFeatureVector(ingredients, ingredientSet):
	featureVector = [0] * len(ingredientSet)
	for index, ingredient in enumerate(ingredientSet):
		if ingredient in ingredients:
			featureVector[index] = 1
		else:
			featureVector[index] = 0
	return featureVector

def getIngreds(file1='ingredients_stats'):
	ingredientSet = set()
	with open(file1, 'rb') as f:
		for line in f:
			ingredientSet.add(line.split()[0])
	return ingredientSet

def predictCuisine(trainX, trainY):
	logreg = linear_model.LogisticRegression()
	logreg.fit(trainX, trainY)
	return logreg

trainX, trainY, ingredientSet, cuisineMap, trainIngredientsList = process("./data.csv")
predictor = predictCuisine(trainX, trainY)
recipeIngredientsList = readIngredients(predictor, ingredientSet)


import os
import pdb
import csv
import numpy as np
import cv2
import torch
import pandas as pd
from tqdm import tqdm
import scipy.io as sio
from spconv.pytorch.utils import PointToVoxel
data_path = "xxx"
save_path = "xxx"
video_files = os.listdir(data_path)

def transform_points_to_voxels(data_dict = {}, device = torch.device("cuda:0")):
    points = data_dict['points']
    # 将points打乱
    shuffle_idx = np.random.permutation(points.shape[0])
    points = points[shuffle_idx]
    data_dict['points'] = points

    voxel_generator = PointToVoxel(
        # 给定每个voxel的长宽高
        vsize_xyz=[10, 10, 10],
        # 给定的范围
        coors_range_xyz=[0, 0, 0, 200, 239, 179],
        # 给定每个event的特征维度 t,x,y,p
        num_point_features=4,
        # 最多选取多少个voxel，
        max_num_voxels=10000,
        # 给定每个voxel中有采样多少个点，不够则补0
        max_num_points_per_voxel=8,  # 32
        device=device
    )
    # 使用spconv生成voxel输出
    points = torch.tensor(data_dict['points'])
    points = points.to(device)
    voxel_output = voxel_generator(points)

    voxels, coordinates, num_points = voxel_output
    # pdb.set_trace()
    voxels = voxels.to(device)
    coordinates = coordinates.to(device)
    num_points = num_points.to(device)
    # 选event数量在前5000的voxel
    if num_points.shape[0] < 512:
        features = voxels[:,:,3]
        coor = coordinates[:,:]
    else:
        _, voxels_idx = torch.topk(num_points, 512)
    # 将每个voxel的1024个p拼接作为voxel初始特征
        features = voxels[voxels_idx][:, :, 3]
    # 前5000个voxel的三维坐标
        coor = coordinates[voxels_idx]
    # 将y.x.t改为t,x,y
    coor[:, [0, 1, 2]] = coor[:, [2, 1, 0]]

    return coor, features

if __name__ == '__main__':
    device = torch.device("cuda:0")
    video_files = os.listdir(data_path)

    label_dict = {lbl: i for i, lbl in enumerate(video_files)}
    for videoID in range(len(video_files)):
        videoName = video_files[videoID]
        fileLIST = os.listdir(os.path.join(data_path, videoName))
        if not os.path.exists(os.path.join(save_path,  videoName)):
            os.makedirs(os.path.join(save_path,  videoName))
        mat_save = os.path.join(save_path,  videoName)
        for FileID in tqdm(range(len(fileLIST))):
            file = fileLIST[FileID]
            save_name = file.split('.')[0]
            if os.path.exists(os.path.join(mat_save,'{}.mat'.format(save_name))):
                continue
            label = label_dict.get(videoName, None)
            read_path = os.path.join(data_path, videoName, file)
            mat_file = sio.loadmat(read_path)
            t = mat_file["ts"]
            time_length = t[-1]-t[0]
            t = ((t-t[0]) / time_length) * 200.0
            x = mat_file["x"].astype(np.float64)
            y = mat_file["y"].astype(np.float64)
            p = mat_file["pol"].astype(np.float64)
            p[np.where(p==0)]=-1
            all_events = np.hstack((t,x,y,p))
            all_events = torch.from_numpy(all_events).float()
            all_events = all_events.to(device)
            data_dict = {'points': all_events}
            coor, features = transform_points_to_voxels(data_dict=data_dict, device=device)
            coor = coor.cpu()
            features = features.cpu()
            coor = coor.numpy()
            features = features.numpy()
            sio.savemat(os.path.join(mat_save , '{}.mat'.format(save_name)), mdict={'coor': coor, 'features': features})


from pyexpat import model
from tabnanny import verbose
from django.db import models

# Create your models here.
class Post( models.Model ):
    titulo = models.CharField(
        max_length=194,
        verbose_name="Título"
    )

    conteudo = models.TextField(
        verbose_name="Conteúdo"
    )

    categoria = models.CharField(
        choices=[
            ('django', 'django'),
            ('python', 'python')
            ],
        default='django',
        verbose_name="Categoria",
        max_length=6
    )

    slug = models.CharField(
        verbose_name="Slug",
        max_length=194,
        blank=False,
    )

    data_publicado = models.DateTimeField(
        auto_now_add=True,
        verbose_name="Data de publicação"
    )

    class Meta:
        verbose_name = "Post"
        verbose_name_plural = "Posts"
        db_table = "post"

    def __str__(self):
        return self.titulo