import numpy as np
from six import string_types
import tensorflow as tf

DEFAULT_PADDING = 'SAME'

def layer(tf_op):
    '''
    Python装饰器的用法，具体教程看这个链接：
    http://wiki.jikexueyuan.com/project/explore-python/Functional/decorator.html
    装饰器的作用是：封装成可以组装的基本网络层（卷积、膨胀卷积、池化等），方便组装复杂网络
    '''
    def layer_decorated(self, *args, **kwargs):
        '''
        装饰器用于装饰(包裹、封装)原有函数的输出，返回的是包装后的函数layer_decorated
        需要装饰的函数，在其函数名上方追加装饰器@layer
        '''
        # 如果没有提供‘name’参数，自动根据操作名称提供一个unique名称
        name = kwargs.setdefault('name', self.get_unique_name(tf_op.__name__))
        # 提取操作层的输入
        if len(self.terminals) == 0:
            raise RuntimeError('No input variables found for layer %s.' % name)
        elif len(self.terminals) == 1:
            layer_input = self.terminals[0]
        else:
            layer_input = list(self.terminals)
        # 调用原函数，获得操作层的输出
        layer_output = tf_op(self, layer_input, *args, **kwargs)
        # 将输出存入类中的self.layers变量
        self.layers[name] = layer_output
        # 把当前输出缓存为下一层的输入
        self.feed(layer_output)
        # 返回类对象本身，以实现链式调用
        return self

    # 返回包装函数layer_decorated
    return layer_decorated

class Network(object):
    '''
    基础网络类，提供基本操作
    '''
    def __init__(self, inputs, trainable=True, is_training=False, num_classes=21):
        # 网络的输入节点
        self.inputs = inputs
        # 当前操作层的输出终端节点list，下一个操作层的输入缓存
        self.terminals = []
        # 操作层缓存器，存储网络输入节点和各层的输出节点
        self.layers = dict(inputs)
        # 是否训练
        self.trainable = trainable
        # dropout的开关变量
        self.use_dropout = tf.placeholder_with_default(tf.constant(1.0),
                                                       shape=[],
                                                       name='use_dropout')
        # 开始构建网络
        self.setup(is_training, num_classes)

    def setup(self, is_training):
        '''网络构建函数，需要子类做实现 '''
        raise NotImplementedError('Must be implemented by the subclass.')

    def load(self, data_path, session, ignore_missing=False):
        '''加载网络权重参数
        data_path: numpy-serialized网络权重参数的文件路径
        session: 当前的 TensorFlow session
        ignore_missing: 是否忽略丢失层的serialized权重参数
        '''
        # 读取numpy-serialized网络权重参数的文件
        data_dict = np.load(data_path).item()
        # 遍历scope
        for op_name in data_dict:
            with tf.variable_scope(op_name, reuse=True):
                # 遍历权重参数名称和权重参数值
                for param_name, data in data_dict[op_name].iteritems():
                    try:
                        # 获取权重参数变量
                        var = tf.get_variable(param_name)
                        # 赋值权重参数变量
                        session.run(var.assign(data))
                    except ValueError:
                        if not ignore_missing:
                            raise

    def feed(self, *args):
        '''Set the input(s) for the next operation by replacing the terminal nodes.
        The arguments can be either layer names or the actual layers.
        '''
        assert len(args) != 0
        # 清空操作层的输入缓存
        self.terminals = []
        # 将函数输入存入 输入缓存
        for fed_layer in args:
            if isinstance(fed_layer, string_types):
                try:
                    fed_layer = self.layers[fed_layer]
                except KeyError:
                    raise KeyError('Unknown layer name fed: %s' % fed_layer)
            # 输入缓存
            self.terminals.append(fed_layer)
        # 返回类对象
        return self

    def get_output(self):
        '''获取当前操作时刻的网络输出节点'''
        return self.terminals[-1]

    def get_unique_name(self, prefix):
        '''基于prefix输入名称，生成unique名称
        '''
        # 为prefix输入名称（例如 conv），计算索引值（例如 1）
        ident = sum(t.startswith(prefix) for t, _ in self.layers.items()) + 1
        # 在prefix输入名称后，追加索引值名称（例如 conv_1）
        return '%s_%d' % (prefix, ident)

    def make_var(self, name, shape):
        '''新建新的TensorFlow变量.'''
        return tf.get_variable(name, shape, trainable=self.trainable)

    def validate_padding(self, padding):
        '''验证边界补值类型：补零，不补值'''
        assert padding in ('SAME', 'VALID')

    @layer
    def conv(self,
             input,
             k_h,
             k_w,
             c_o,
             s_h,
             s_w,
             name,
             relu=True,
             padding=DEFAULT_PADDING,
             group=1,
             biased=True):
        '''2D卷积'''

        # 验证padding类型
        self.validate_padding(padding)
        # 获取输入通道数量
        c_i = input.get_shape().as_list()[-1]
        # 验证group参数
        assert c_i % group == 0
        assert c_o % group == 0
        # 基于不同的输入和卷积核，定义卷积函数
        convolve = lambda i, k: tf.nn.conv2d(i, k, [1, s_h, s_w, 1], padding=padding)
        with tf.variable_scope(name) as scope:
            # 为卷积核，新建权重参数
            kernel = self.make_var('weights', shape=[k_h, k_w, c_i / group, c_o])
            if group == 1:
                # 普通卷积
                output = convolve(input, kernel)
            else:
                # 拆分通道，进行分组卷积
                input_groups = tf.split(3, group, input)
                kernel_groups = tf.split(3, group, kernel)
                output_groups = [convolve(i, k) for i, k in zip(input_groups, kernel_groups)]
                # 串接分组卷积结果
                output = tf.concat(3, output_groups)
            # 添加篇置
            if biased:
                biases = self.make_var('biases', [c_o])
                output = tf.nn.bias_add(output, biases)
            if relu:
                # ReLU 激活函数
                output = tf.nn.relu(output, name=scope.name)
            return output

    @layer
    def atrous_conv(self,
                    input,
                    k_h,
                    k_w,
                    c_o,
                    dilation,
                    name,
                    relu=True,
                    padding=DEFAULT_PADDING,
                    group=1,
                    biased=True):
        '''2D膨胀卷积'''

        # 验证padding类型
        self.validate_padding(padding)
        # 获取输入通道数量
        c_i = input.get_shape().as_list()[-1]
        # 验证group参数
        assert c_i % group == 0
        assert c_o % group == 0
        # 基于不同的输入和卷积核，定义膨胀卷积函数
        convolve = lambda i, k: tf.nn.atrous_conv2d(i, k, dilation, padding=padding)
        with tf.variable_scope(name) as scope:
            # 为卷积核，新建权重参数
            kernel = self.make_var('weights', shape=[k_h, k_w, c_i / group, c_o])
            if group == 1:
                # 普通卷积
                output = convolve(input, kernel)
            else:
                # 拆分通道，进行分组膨胀卷积
                input_groups = tf.split(3, group, input)
                kernel_groups = tf.split(3, group, kernel)
                output_groups = [convolve(i, k) for i, k in zip(input_groups, kernel_groups)]
                # 串接分组膨胀卷积结果
                output = tf.concat(3, output_groups)
            # 添加篇置
            if biased:
                biases = self.make_var('biases', [c_o])
                output = tf.nn.bias_add(output, biases)
            if relu:
                # ReLU 激活函数
                output = tf.nn.relu(output, name=scope.name)
            return output

    @layer
    def relu(self, input, name):
        '''ReLU激活'''
        return tf.nn.relu(input, name=name)

    @layer
    def max_pool(self, input, k_h, k_w, s_h, s_w, name, padding=DEFAULT_PADDING):
        '''最大池化'''
        self.validate_padding(padding)
        return tf.nn.max_pool(input,
                              ksize=[1, k_h, k_w, 1],
                              strides=[1, s_h, s_w, 1],
                              padding=padding,
                              name=name)

    @layer
    def avg_pool(self, input, k_h, k_w, s_h, s_w, name, padding=DEFAULT_PADDING):
        '''均值池化'''
        self.validate_padding(padding)
        return tf.nn.avg_pool(input,
                              ksize=[1, k_h, k_w, 1],
                              strides=[1, s_h, s_w, 1],
                              padding=padding,
                              name=name)

    @layer
    def lrn(self, input, radius, alpha, beta, name, bias=1.0):
        '''局部响应归一化'''
        return tf.nn.local_response_normalization(input,
                                                  depth_radius=radius,
                                                  alpha=alpha,
                                                  beta=beta,
                                                  bias=bias,
                                                  name=name)

    @layer
    def concat(self, inputs, axis, name):
        '''通道串接'''
        return tf.concat(concat_dim=axis, values=inputs, name=name)

    @layer
    def add(self, inputs, name):
        '''逐像素相加'''
        return tf.add_n(inputs, name=name)

    @layer
    def fc(self, input, num_out, name, relu=True):
        '''全连接层'''
        with tf.variable_scope(name) as scope:
            input_shape = input.get_shape()
            if input_shape.ndims == 4:
                # The input is spatial. Vectorize it first.
                dim = 1
                for d in input_shape[1:].as_list():
                    dim *= d
                feed_in = tf.reshape(input, [-1, dim])
            else:
                feed_in, dim = (input, input_shape[-1].value)
            weights = self.make_var('weights', shape=[dim, num_out])
            biases = self.make_var('biases', [num_out])
            op = tf.nn.relu_layer if relu else tf.nn.xw_plus_b
            fc = op(feed_in, weights, biases, name=scope.name)
            return fc

    @layer
    def softmax(self, input, name):
        '''softmax层'''
        input_shape = map(lambda v: v.value, input.get_shape())
        if len(input_shape) > 2:
            # For certain models (like NiN), the singleton spatial dimensions
            # need to be explicitly squeezed, since they're not broadcast-able
            # in TensorFlow's NHWC ordering (unlike Caffe's NCHW).
            if input_shape[1] == 1 and input_shape[2] == 1:
                input = tf.squeeze(input, squeeze_dims=[1, 2])
            else:
                raise ValueError('Rank 2 tensor input expected for softmax!')
        return tf.nn.softmax(input, name)

    @layer
    def batch_normalization(self, input, name, is_training, activation_fn=None, scale=True):
        '''BN层'''
        with tf.variable_scope(name) as scope:
            output = tf.contrib.layers.batch_norm(
                input,
                activation_fn=activation_fn,
                is_training=is_training,
                updates_collections=None,
                scale=scale,
                scope=scope)
            return output

    @layer
    def dropout(self, input, keep_prob, name):
        '''dropout层'''
        keep = 1 - self.use_dropout + (self.use_dropout * keep_prob)
        return tf.nn.dropout(input, keep, name=name)


from ..io.reader import BinaryFileReader
from .rsm import Rsm
from pprint import pprint

class RsmReader(object):
    def __init__(self):
        pass

    @staticmethod
    def from_file(path) -> Rsm:
        with open(path, 'rb') as f:
            rsm = Rsm()
            reader = BinaryFileReader(f)
            magic = reader.read('4s')[0]
            if magic != b'GRSM':
                raise RuntimeError('Invalid file type')
            version_major, version_minor = reader.read('2B')
            rsm.anim_length = reader.read('I')
            rsm.shade_type = reader.read('I')
            if version_major > 1 or version_major and version_minor >= 4:
                rsm.alpha = reader.read('B')
            reader.read('16B')
            texture_count = reader.read('I')[0]
            for i in range(texture_count):
                texture = reader.read_fixed_length_null_terminated_string(40)
                rsm.textures.append(texture)
            rsm.main_node = reader.read_fixed_length_null_terminated_string(40)
            node_count = reader.read('I')[0]
            for i in range(node_count):
                node = rsm.Node()
                node.name = reader.read_fixed_length_null_terminated_string(40)
                node.parent_name = reader.read_fixed_length_null_terminated_string(40)
                node_texture_count = reader.read('I')[0]
                node.texture_indices = reader.read('{}I'.format(node_texture_count))
                node.some_matrix = reader.read('9f') # offset matrix??
                node.offset_ = reader.read('3f')
                node.offset = reader.read('3f')
                node.rotation = reader.read('4f')  # axis-angle
                node.scale = reader.read('3f')
                vertex_count = reader.read('I')[0]
                node.vertices = [reader.read('3f') for _ in range(vertex_count)]
                # Texture Coordinates
                texcoord_count = reader.read('I')[0]
                for _ in range(texcoord_count):
                    texcoord = reader.read('I2f')
                    node.texcoords.append(texcoord)
                # Faces
                face_count = reader.read('I')[0]
                for _ in range(face_count):
                    face = Rsm.Node.Face()
                    face.vertex_indices = reader.read('3H')
                    face.texcoord_indices = reader.read('3H')
                    face.texture_index = reader.read('H')[0]
                    reader.read('H') # padding
                    face.two_sided = reader.read('I')[0]
                    face.smoothing_group = reader.read('I')[0]
                    node.faces.append(face)
                rsm.nodes.append(node)
                if version_major > 1 or (version_major == 1 and version_minor >= 5):
                    location_keyframe_count = reader.read('I')[0]
                    for _ in range(location_keyframe_count):
                        location_keyframe = Rsm.Node.LocationKeyFrame()
                        location_keyframe.frame = reader.read('I')
                        location_keyframe.position = reader.read('3f')
                        node.location_keyframes.append(location_keyframe)
                rotation_keyframe_count = reader.read('I')[0]
                for _ in range(rotation_keyframe_count):
                    rotation_keyframe = Rsm.Node.RotationKeyFrame()
                    rotation_keyframe.frame = reader.read('I')
                    rotation_keyframe.rotation = reader.read('4f')
                    node.rotation_keyframes.append(rotation_keyframe)
            return rsm


import os
import gym
import random
import argparse
import numpy as np
import tensorflow as tf
from networks import CartPoleNetwork
from self_play import self_play
from replay import ReplayBuffer
from config import get_cartpole_config

gpus = tf.config.experimental.list_physical_devices('GPU')
for gpu in gpus:
    tf.config.experimental.set_memory_growth(gpu, True)

SEED = 10

def set_seeds(seed=SEED):
    os.environ['PYTHONHASHSEED'] = str(seed)
    random.seed(seed)
    np.random.seed(seed)
    tf.random.set_seed(seed)

if __name__ == "__main__":
    with tf.device('/GPU:0'):
        parser = argparse.ArgumentParser(description='Train MuZero.')
        parser.add_argument('--num_simulations', type=int, default=50)
        parser.add_argument('--save_dir', type=str, default="tmp_results")
        parser.add_argument('--seed', type=int, default=SEED)
        args = parser.parse_args()
        print("num simulations:", args.num_simulations)

        # Set seeds for reproducibility
        set_seeds(args.seed)
        # Create the cartpole network
        network = CartPoleNetwork(
            action_size=2, state_shape=(None, 4), embedding_size=4, max_value=200)
        print("Network Made")
        # Set the configuration for muzero
        config = get_cartpole_config(args.num_simulations)  # Create Environment
        env = gym.make('CartPole-v0')

        # Create buffer to store games
        replay_buffer = ReplayBuffer(config)
        print(args)
        os.makedirs(args.save_dir, exist_ok=True)
        self_play(env, config, replay_buffer, network, save_dir=args.save_dir)


from flask import request, jsonify
from ..model.ItemOrcamento import ItemOrcamento, itemOrcamento_schema, itemOrcamentos_schema
from ..model.ItemOrcamento import db

def cad_itemOrcamento():
    resp = request.get_json()
    nome = resp['nome']
    valor = resp['valor']

    item = ItemOrcamento(nome=nome, valor=valor)

    try:
        db.session.add(item)
        db.session.commit()
        return jsonify({'msg': 'Cadastrado com sucesso'}), 201
    except Exception as e:
        db.session.rollback()
        return jsonify({'msg': 'Erro ao cadastrar ItemOrcamento', 'error': str(e)}), 500

def altera_itemOrcamento(id):
    item = ItemOrcamento.query.get(id)
    if item:
        resp = request.get_json()
        try:
            item.nome = resp['nome']
            item.valor = resp['valor']
            db.session.commit()
            return jsonify({'msg': 'Alterado com sucesso', 'dados': itemOrcamento_schema.dump(item)}), 200
        except Exception as e:
            db.session.rollback()
            return jsonify({'msg': 'Não foi possível alterar', 'dados': {}, 'error': str(e)}), 500
    else:
        return jsonify({'msg': 'Item não encontrado', 'dados': {}, 'error': str(e)}), 404

def excluir_itemOrcamento(id):
    item = ItemOrcamento.query.get(id)
    if item:
        try:
            db.session.delete(item)
            db.session.commit()
            return jsonify({'msg': 'Excluido com sucesso', 'dados': itemOrcamento_schema.dump(item)}), 200
        except Exception as e:
            db.session.rollback()
            return jsonify({'msg': 'Não foi possivel excluir', 'dados': {}, 'error': str(e)}), 500
    else:
        return jsonify({'msg': 'Item não encontrado', 'dados': {}, 'error': str(e)}), 404

def buscar_todos():
    try:
        itens = ItemOrcamento.query.all()
        return jsonify({'msg': 'Busca efetuada com sucesso', 'dados': itemOrcamentos_schema.dump(itens)}), 200
    except Exception as e:
        db.session.rollback()
        return jsonify({'msg': 'Não foi possivel efetuar a busca', 'dados': {}, 'error': str(e)}), 500


import torch
import torch.nn as nn
import torch.nn.functional as F

from ..common.blocks import Conv2dReLU
from ..base.model import Model
from ..encoders.scse import SCse

class TransposeX2(nn.Module):

    def __init__(self, in_channels, out_channels, use_batchnorm=True, **batchnorm_params):
        super().__init__()
        layers = []
        layers.append(nn.ConvTranspose2d(in_channels, out_channels, kernel_size=4, stride=2, padding=1))
        if use_batchnorm:
            layers.append(nn.BatchNorm2d(out_channels, **batchnorm_params))
        layers.append(nn.ReLU(inplace=True))

        self.block = nn.Sequential(*layers)

    def forward(self, x):
        return self.block(x)

class DecoderBlock(nn.Module):
    def __init__(self, in_channels, out_channels, use_batchnorm=True, se_module=False):
        super().__init__()
        if se_module:
            self.block = nn.Sequential(
                Conv2dReLU(in_channels, in_channels // 4, kernel_size=1, use_batchnorm=use_batchnorm),
                TransposeX2(in_channels // 4, in_channels // 4, use_batchnorm=use_batchnorm),
                Conv2dReLU(in_channels // 4, out_channels, kernel_size=1, use_batchnorm=use_batchnorm),
                SCse(out_channels)
            )
        else:
            self.block = nn.Sequential(
                Conv2dReLU(in_channels, in_channels // 4, kernel_size=1, use_batchnorm=use_batchnorm),
                TransposeX2(in_channels // 4, in_channels // 4, use_batchnorm=use_batchnorm),
                Conv2dReLU(in_channels // 4, out_channels, kernel_size=1, use_batchnorm=use_batchnorm),
            )

    def forward(self, x):
        x, skip = x
        x = self.block(x)
        if skip is not None:
            x = x + skip
        return x

class LinknetDecoder(Model):

    def __init__(
            self,
            encoder_channels,
            prefinal_channels=32,
            final_channels=1,
            use_batchnorm=True,
            se_module=False,
            h_columns=False
    ):
        super().__init__()

        in_channels = encoder_channels
        self.h_columns = h_columns

        self.layer1 = DecoderBlock(in_channels[0], in_channels[1], use_batchnorm=use_batchnorm, se_module=se_module)
        self.layer2 = DecoderBlock(in_channels[1], in_channels[2], use_batchnorm=use_batchnorm, se_module=se_module)
        self.layer3 = DecoderBlock(in_channels[2], in_channels[3], use_batchnorm=use_batchnorm, se_module=se_module)
        self.layer4 = DecoderBlock(in_channels[3], in_channels[4], use_batchnorm=use_batchnorm, se_module=se_module)
        self.layer5 = DecoderBlock(in_channels[4], prefinal_channels, use_batchnorm=use_batchnorm, se_module=se_module)

        if self.h_columns:
            self.layer1_h = nn.Conv2d(in_channels[1], prefinal_channels, kernel_size=(1, 1))
            self.layer2_h = nn.Conv2d(in_channels[2], prefinal_channels, kernel_size=(1, 1))
            self.layer3_h = nn.Conv2d(in_channels[3], prefinal_channels, kernel_size=(1, 1))
            self.layer4_h = nn.Conv2d(in_channels[4], prefinal_channels, kernel_size=(1, 1))
            self.final_conv = nn.Sequential(nn.Conv2d(int(prefinal_channels * 5), 64, kernel_size=3, padding=1),
                                            nn.ELU(True),
                                            nn.Conv2d(64, 1, kernel_size=1, bias=False))
        else:
            self.final_conv = nn.Conv2d(prefinal_channels, final_channels, kernel_size=(1, 1))

        self.initialize()

    def forward(self, x):
        encoder_head = x[0]
        skips = x[1:]

        if self.h_columns:
            d5 = self.layer1([encoder_head, skips[0]])
            d4 = self.layer2([d5, skips[1]])
            d3 = self.layer3([d4, skips[2]])
            d2 = self.layer4([d3, skips[3]])
            d1 = self.layer5([d2, None])
            x = torch.cat((d1,
                           self.layer4_h(F.upsample(d2, scale_factor=2, mode='bilinear', align_corners=True)),
                           self.layer3_h(F.upsample(d3, scale_factor=4, mode='bilinear', align_corners=True)),
                           self.layer2_h(F.upsample(d4, scale_factor=8, mode='bilinear', align_corners=True)),
                           self.layer1_h(F.upsample(d5, scale_factor=16, mode='bilinear', align_corners=True))), 1)
        else:
            x = self.layer1([encoder_head, skips[0]])
            x = self.layer2([x, skips[1]])
            x = self.layer3([x, skips[2]])
            x = self.layer4([x, skips[3]])
            x = self.layer5([x, None])
        x = self.final_conv(x)

        return x


from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.common.action_chains import ActionChains

PATH = r"C:\Program Files (x86)\chromedriver.exe"  # Path of the selenium driver
s = Service(executable_path=PATH)
driver = webdriver.Chrome(service=s)  # Every selenium file needs above lines (For my computer :D)

driver.get("https://orteil.dashnet.org/cookieclicker/")  # URL of the game

driver.implicitly_wait(5)

the_cookie = driver.find_element(By.ID, "bigCookie")
cookie_counter = driver.find_element(By.ID, "cookies")

items_to_buy = [driver.find_element(By.ID, "productPrice1"), driver.find_element(By.ID, "productPrice0")]  # Items can be modified

for i in range(1000):
    actions = ActionChains(driver)
    actions.click(the_cookie)
    actions.perform()

    count = int(cookie_counter.text.split(" ")[0])

    for item_to_buy in items_to_buy:
        price = int(item_to_buy.text)

        buy_action = ActionChains(driver)  # Action buffer
        buy_action.click(item_to_buy)  # click to buy

        if count >= price:
            buy_action.perform()  # Buy the upgrade!


import pathlib
from typing import List

import cv2
import h5py
import numpy as np
import typer

def split_text(txt: List[np.bytes_]) -> List[List[str]]:
    """テキストを文字単位に分割

    Args:
        txt (List[np.bytes_]): テキスト文字列の配列

    Returns:
        List[List[str]]: テキスト文字列の文字単位に分割した配列
    """

    txt = [i.decode('utf-8') for i in txt]

    splitted_text = []

    for words in txt:
        for word in words.split('\n'):
            splitted_char = []
            for char in list(word):
                if char != ' ':
                    splitted_char.append(char)
            splitted_text.append(splitted_char)

    return splitted_text

def convert_bbox(imgaug_bboxes):
    all_bbox = []

    for i in imgaug_bboxes.bounding_boxes:
        bbox = i.to_keypoints()
        all_bbox.append(np.array([
            [bbox[0].x, bbox[0].y],
            [bbox[1].x, bbox[1].y],
            [bbox[2].x, bbox[2].y],
            [bbox[3].x, bbox[3].y]
        ]))

    return np.stack(all_bbox)

def main(file_path: str = 'src/dataset/SynthText.h5', saved_path: str = 'src/dataset/synthtext'):

    pathlib.Path(saved_path).mkdir(exist_ok=True)
    pathlib.Path(f'{saved_path}/char_bbox').mkdir(exist_ok=True)
    pathlib.Path(f'{saved_path}/image').mkdir(exist_ok=True)
    pathlib.Path(f'{saved_path}/text').mkdir(exist_ok=True)

    with h5py.File(file_path, 'r') as db:

        dataset = sorted(db['data'].keys())

        for image_name in dataset:

            image = db['data'][image_name][...]
            char_bbox = db['data'][image_name].attrs['charBB']
            txt = db['data'][image_name].attrs['txt']

            splitted_text = split_text(txt)

            char_bbox = char_bbox.transpose(2, 1, 0)

            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)

            cv2.imwrite(f"{saved_path}/image/{image_name}", image)
            np.save(f"{saved_path}/char_bbox/{image_name.replace('.jpg', '')}.npy", char_bbox)
            with open(f"{saved_path}/text/{image_name.replace('.jpg', '')}.txt", 'w') as f:
                for x in splitted_text:
                    f.write(str(x) + "\n")

if __name__ == "__main__":
    typer.run(main)


import codecs
import os
from setuptools import setup

from release_exporter.version import __version__

here = os.path.abspath(os.path.dirname(__file__)) + os.sep

def get_requirements(*parts):
    return codecs.open(os.path.join(here, *parts), 'r').read().splitlines()

setup(
    name='release-exporter',
    version=__version__,
    install_requires=get_requirements('requirements.txt'),
    packages=['tests', 'release_exporter'],
    url='https://www.gollahalli.com/blog/export-your-github-and-gitlab-releases-as-a-changelog/',
    license='MIT',
    author='Akshay Raj Gollahalli',
    author_email='akshay@gollahalli.com',
    description='Release exporter for GitHub and GitLab.',
    long_description=open('README.md').read(),
    long_description_content_type='text/markdown',
    keywords=['changelog', 'releases'],
    classifiers=[
        'License :: OSI Approved :: MIT License',
        'Programming Language :: Python :: 3 :: Only',
        'Programming Language :: Python :: 3.8',
        'Programming Language :: Python :: 3.9',
        'Programming Language :: Python :: 3.10',
        'Programming Language :: Python :: 3.11',
        'Intended Audience :: Developers',
        'Environment :: Console',
        'Development Status :: 5 - Production/Stable',
        'Topic :: Utilities'
    ],
    entry_points={
        'console_scripts': [
            'rex = release_exporter.cli:main'
        ]
    }
)


import os,sys
import numpy as np
import matplotlib
matplotlib.use('TkAgg')
import matplotlib.pyplot as plt

import pickle
from baselines import logger
import ppo
import gym
from train_ppo import *

import globals
from time import *
from utils.plotting_performance import *
from utils.utils import *

from brs_engine.DubinsCar_brs_engine import *
from brs_engine.PlanarQuad_brs_engine import *

# ranges: [(1,10), (20,30), (40,50), ..]
def gen_plots(args, ranges):
    if args['loadpath'] == None:
        raise ValueError("Loading path is None!!")
    if args['gym_env'] == "PlanarQuadEnv-v0":
        lo = [-5, 0, -np.pi]
        hi = [5, 10, np.pi]
        bins = [100, 100, 100]
    elif args['gym_env'] == "DubinsCarEnv-v0":
        lo = [-5, -5, 0]
        hi = [5, 5, np.pi * 2]
        bins = [100, 100, 100]
    else:
        raise ValueError("env's name is invalid!!")

    full_dir = args['loadpath']+'/heatmap'
    num_files = len([x for x in os.listdir(full_dir)])
    # print("num files:", num_files)

    # fig = plt.figure()

    num_fig = len(ranges)
    fig, axes = plt.subplots(2, num_fig)
    print(axes)
    for rg in ranges:
        if len(rg) != 2 or rg[0] > rg[1] or rg[0] > num_files or rg[1] > num_files:
            raise ValueError("Please provide valid range series!!")
        else:
            rg_lo = rg[0]
            rg_hi = rg[1]
            all_obs_cur_rg = []
            all_dones_cur_rg = []
            for id in range(rg_lo, rg_hi):
                with open(full_dir + '/iter_' + str(id) + '.pkl', 'rb') as f:
                    seg_info = pickle.load(f)
                # obs: np.array([[_,_,_],
                #                [_,_,_]])
                all_obs_cur_rg.append(seg_info['ob'])
            all_obs_cur_rg_concat = np.concatenate(all_obs_cur_rg, axis=0)
            if args['gym_env'] == "PlanarQuadEnv-v0":
                x = all_obs_cur_rg_concat[:, 0]
                z = all_obs_cur_rg_concat[:, 2]
                theta = all_obs_cur_rg_concat[:,4]
                # ax = fig.add_subplot(1,num_fig,1+ranges.index(rg), adjustable='box', aspect=1.0)

                # ax.set_xlabel("X position")
                # ax.set_ylabel("Z position")
                # ax.hist2d(x, z, bins=min(bins[:2]), range=[[-5, 5], [0, 10]])
                # ax.hist2d(x, z, bins=min(bins[:2]))

                # theta as 1d hist, xz as 2d hist
                axes[0, ranges.index(rg)].set(adjustable='box', aspect='equal')
                axes[0, ranges.index(rg)].hist(theta, bins=bins[2], density=True, facecolor='g', alpha=0.75, log=True)
                axes[1, ranges.index(rg)].set(adjustable='box', aspect='equal')
                axes[1, ranges.index(rg)].hist2d(x, z, range=[[-5,5],[0,10]], bins=min(bins[:2]), cmap='tab20')

                # x as 1d hist, z&theta as 2d hist
                # axes[0, ranges.index(rg)].set(adjustable='box', aspect='equal')
                # axes[0, ranges.index(rg)].hist(x, bins=bins[0], density=True, facecolor='g', alpha=0.75, log=True)
                # axes[1, ranges.index(rg)].set(adjustable='box', aspect='equal')
                # axes[1, ranges.index(rg)].hist2d(z,theta, range=[[0,10],[-np.pi,np.pi]], bins=min(bins[1:]), cmap='tab20')

                # z as 1d hist, x&theta as 2d hist
                # axes[0, ranges.index(rg)].set(adjustable='box', aspect='equal')
                # axes[0, ranges.index(rg)].hist(z, bins=bins[1], density=True, facecolor='g', alpha=0.75, log=True)
                # axes[1, ranges.index(rg)].set(adjustable='box', aspect='equal')
                # axes[1, ranges.index(rg)].hist2d(x, theta, range=[[-5, 5], [-np.pi, np.pi]], bins=bins[0],
                #                                  cmap='tab20')
            elif args['gym_env'] == "DubinsCarEnv-v0":
                xy = all_obs_cur_rg_concat[:, :2]
                ax = fig.add_subplot(1,num_fig,1+ranges.index(rg))
                ax.set_xlabel('X position')
                ax.set_ylabel('Y position')
                ax.hist2d(xy[:, 0], xy[:, 1], bins=min(bins[:2]))
            else:
                ValueError("env's name is invalid!!")
    # for ax in axes:
    #     ax.set_xlabel('Common x-label')
    #     ax.set_ylabel('Common y-label')
    # fig.add_subplot(111, frameon=False)
    # plt.tick_params(labelcolor='none', top=False, bottom=False, left=False, right=False)
    # plt.xlabel('Common x-label')
    # plt.ylabel('Common z-label')
    fig.tight_layout()
    plt.show()

    plt.savefig(args['loadpath'] + '/test3_xy.png')
if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--gym_env", help="which gym environment to use.", type=str, default='PlanarQuadEnv-v0')
    parser.add_argument("--loadpath", help="the path you load for generating heatmaps", type=str, default=None)
    args = parser.parse_args()
    args = vars(args)

    # gen_plots(args, [(1,20), (20,40), (40,60), (60,80), (80,100), (100,120)])
    gen_plots(args, [(1,20), (20,40), (40,60)])

# -----------  this part below is for generating obs during training for building heatmaps later ---------------------
# globals.init()
# def gen_iter_obs(args):
#     assert args['gym_env'] == 'DubinsCarEnv-v0' or args['gym_env'] == 'PlanarQuadEnv-v0'
#     logger.configure(dir=args['RUN_DIR'])
#     globals.g_hm_dirpath = args['HEATMAP_DIR']
#     globals.g_iter_id = 0
#
#     env = gym.make(args['gym_env'])
#     env.reward_type = args['reward_type']
#     env.set_additional_goal = args['set_additional_goal']
#     logger.record_tabular("algo", args['algo'])
#     logger.record_tabular("env", args['gym_env'])
#     logger.record_tabular("env.set_additional_goal", args['set_additional_goal'])
#     logger.record_tabular("env.reward_type", args['reward_type'])
#
#     logger.dump_tabular()
#
#
#     # Initialize brs engine. You also have to call reset_variables() after instance initialization
#     if args['reward_type'] == 'ttr':
#         if args['gym_env'] == 'DubinsCarEnv-v0':
#             brsEngine = DubinsCar_brs_engine()
#             brsEngine.reset_variables()
#         elif args['gym_env'] == 'PlanarQuadEnv-v0':
#             brsEngine = Quadrotor_brs_engine()
#             brsEngine.reset_variables()
#         else:
#             raise ValueError("invalid environment name for ttr reward!")
#         # You have to assign the engine
#         env.brsEngine = brsEngine
#     elif args['reward_type']  in ['hand_craft', 'distance', 'distance_lambda_10', 'distance_lambda_1',
#                               'distance_lambda_0.1']:
#         pass
#     else:
#         raise ValueError("wrong type of reward")
#     ppo_params_json = os.environ['PROJ_HOME'] + '/ppo_params.json'
#
#     # Start to train the policy
#     trained_policy = train(env=env, algorithm=ppo, params=ppo_params_json, args=args, save_obs=True)
#     trained_policy.save_model(args['MODEL_DIR'])
#
#     return True
#
# if __name__ == "__main__":
#     with tf.device('/gpu:1'):
#         # ----- path setting ------
#         parser = argparse.ArgumentParser()
#         parser.add_argument("--gym_env", help="which gym environment to use.", type=str, default='PlanarQuadEnv-v0')
#         parser.add_argument("--reward_type", help="which type of reward to use.", type=str, default='ttr')
#         parser.add_argument("--algo", help="which type of algorithm to use.", type=str, default='ppo')
#         parser.add_argument("--set_additional_goal", type=str, default="angle")
#         args = parser.parse_args()
#         args = vars(args)
#
#         RUN_DIR = os.path.join(os.getcwd(), 'heatmaps_icra',
#                                args['gym_env']+ '_' + args['reward_type'] + '_' + args['algo'] + '_' + strftime(
#                                    '%d-%b-%Y_%H-%M-%S'))
#         MODEL_DIR = os.path.join(RUN_DIR, 'model')
#         FIGURE_DIR = os.path.join(RUN_DIR, 'figure')
#         RESULT_DIR = os.path.join(RUN_DIR, 'result')
#         HEATMAP_DIR = os.path.join(RUN_DIR, 'heatmap')
#
#         maybe_mkdir(RUN_DIR)
#         maybe_mkdir(MODEL_DIR)
#         maybe_mkdir(FIGURE_DIR)
#         maybe_mkdir(RESULT_DIR)
#         maybe_mkdir(HEATMAP_DIR)
#
#         args['RUN_DIR'] = RUN_DIR
#         args['MODEL_DIR'] = MODEL_DIR
#         args['FIGURE_DIR'] = FIGURE_DIR
#         args['RESULT_DIR'] = RESULT_DIR
#         args['HEATMAP_DIR'] = HEATMAP_DIR
#         gen_iter_obs(args)


import mlflow
import sys

sys.path.append("~/torch_study/paper/06_BERT/source")
from utils import *
from finetune.ner_bert import *
from finetune.dataset import *
import torch
import dill
import pytorch_lightning as pl
import mlflow.pytorch
from mlflow.tracking import MlflowClient
from pytorch_lightning.callbacks import ModelCheckpoint
from torch.utils.data import Dataset, DataLoader

class NER_BERT_trainer(pl.LightningModule):
    def __init__(self, config):
        super(NER_BERT_trainer, self).__init__()
        self.config = config
        data_config = config["data"]

        dataset = NER_Dataset(data_config["src"], data_config["vocab"])
        valid_dataset = NER_Dataset(data_config["src_valid"], data_config["vocab"])

        self.pad_idx = dataset.tokenizer.token_to_id("[PAD]")
        dataloader = DataLoader(
            dataset,
            data_config["batch_size"],
            collate_fn=lambda batch: pad_collate(batch, self.pad_idx),
        )

        valid_dataloader = DataLoader(
            valid_dataset,
            data_config["batch_size"],
            collate_fn=lambda batch: pad_collate(batch, self.pad_idx),
        )

        print("before special token", dataset.tokenizer.get_vocab_size())
        dataset.tokenizer.add_special_tokens(["[SEP]", "[CLS]", "[MASK]", "[EOD]"])
        self.vocab_size = dataset.tokenizer.get_vocab_size()
        print("after special token", self.vocab_size)

        self.output_dim = dataset.output_dim
        self.ner_bert = NER_BERT(
            self.config, self.vocab_size, self.output_dim, self.pad_idx
        )
        device = config["train"]["device"]
        self.ner_bert.to(device)
        self.ner_bert.train()
        self.ner_bert.zero_grad()
        self.ner_bert.fcn.apply(self.initialize_weights)

        if device == "cuda":
            self.gpus = 1
        else:
            self.gpus = 0

        artifact_path = mlflow.mlflow.get_artifact_uri()
        artifact_path = artifact_path.replace("file://", "")
        print("////////////////arftifact_path", artifact_path)
        checkpoint_callback = ModelCheckpoint(
            dirpath=artifact_path, monitor="valid_macro_f1"
        )

        # Add your callback to the callbacks list
        trainer = pl.Trainer(
            callbacks=[checkpoint_callback],
            max_epochs=self.config["train"]["n_epochs"],
            progress_bar_refresh_rate=10,
            gpus=self.gpus,
        )

        # Auto log all MLflow entities
        mlflow.pytorch.autolog()

        # Train the model
        mlflow.end_run()  # 이전에 돌아가고 있던거 끄기
        with mlflow.start_run() as run:
            for key, value in config.items():
                mlflow.log_param(key, value)
            trainer.fit(self.ner_bert, dataloader, valid_dataloader)

    # https://github.com/GyuminJack/torchstudy/blob/main/06Jun/BERT/src/trainer.py
    def initialize_weights(self, m):
        for name, param in m.named_parameters():
            if ("fc" in name) or ("embedding" in name):
                if "bias" in name:
                    torch.nn.init.zeros_(param.data)
                else:
                    torch.nn.init.normal_(param.data, mean=0.0, std=0.02)
            elif "layer_norm" in name:
                if "bias" in name:
                    torch.nn.init.zeros_(param.data)
                else:
                    torch.nn.init.constant_(param.data, 1.0)

if __name__ == "__main__":
    config_file = "~/torch_study/paper/06_BERT/config_finetune.yaml"
    config = read_yaml(config_file)
    print("train started..")
    trainer = NER_BERT_trainer(config)


import time
from node_bfs_class import node
import heapq as hq
from PIL import Image
import imageio
import numpy as np
import os

class graph:
    """a container structure for nodes"""

    def __init__(self, map, watch_evolution=True):
        self.map = [x.copy() for x in map]
        self.pixel_array = [[(0,0,0) if val == 'X' else (255,255,255) for val in row] for row in map]
        self.start_node_coords = self.get_start_node_coords()
        self.goal_node_coords = self.get_goal_node_coords()
        self.nodes = {}
        self.active_nodes = {}
        self.watch_evolution = watch_evolution
        self.populate_nodes()

    def get_start_node_coords(self):
        """gets the coordinates of the start node"""
        i = ['S' in row for row in self.map].index(True)
        j = [x == 'S' for x in self.map[i]].index(True)
        return (i, j)

    def get_goal_node_coords(self):
        """gets the coordinates of the goal node"""
        i = ['G' in row for row in self.map].index(True)
        j = [x == 'G' for x in self.map[i]].index(True)
        return (i, j)

    def populate_nodes(self):
        self.nodes = {(i, j): node(self.map,
                      (i, j),
                      (i, j) == self.goal_node_coords,
                      (i, j) == self.start_node_coords)
                      for i in range(len(self.map))
                      for j in range(len(self.map[i]))
                      if self.map[i][j] != 'X'}

    def get_path(self, end):
        ret = []
        n = end
        while self.nodes[n].parent != self.start_node_coords:
            ret.append(n)
            n = self.nodes[n].parent

        ret.append(n)

        return ret[::-1]

    def a_star_search(self):
        """ different approach to A* """
        queue = []
        hq.heappush(queue, (0, self.start_node_coords))

        self.nodes[self.start_node_coords].set_distance_to_goal(self.goal_node_coords)
        self.nodes[self.start_node_coords].set_path_length(0.)

        while queue:
            tup = hq.heappop(queue)
            n = self.nodes[tup[1]]

            if n.loc == self.goal_node_coords:
                self.draw_path(self.get_path(n.loc))
                #self.draw_path_to_image(self.get_path(n.loc), final_path=True)
                return True

            if not n.is_active:
                n.activate()
                n.set_distance_to_start(self.start_node_coords)
                n.set_distance_to_goal(self.goal_node_coords)

            for neighbor in n.neighbors:
                dist = n.path_length + n.distance_to_node(neighbor)
                if dist < self.nodes[neighbor].path_length:
                    self.nodes[neighbor].set_parent(n.loc)
                    self.nodes[neighbor].set_distance_to_goal(self.goal_node_coords)
                    self.nodes[neighbor].set_path_length(dist)
                    if (self.nodes[neighbor].cost, neighbor) not in queue:
                        hq.heappush(queue, (self.nodes[neighbor].cost, neighbor))

            if self.watch_evolution and queue:
                self.draw_path([x[1] for x in queue], reset=True, symb='Q')
                time.sleep(0.005)

            #if queue and len(queue) > 1:
            #    self.draw_path_to_image(queue, final_path=False)

        return False

    def draw_path(self, path, reset=False, symb="P"):
        """draws a path, and resets the board if necessary"""
        changed_coords = []
        snc = self.start_node_coords
        gnc = self.goal_node_coords
        self.map[snc[0]][snc[1]] = "\033[93mS\033[0m"
        self.map[gnc[0]][gnc[1]] = "\033[93mG\033[0m"

        for coord in path:
            if coord != snc and coord != gnc:
                self.map[coord[0]][coord[1]] = "\033[93m"+symb+"\033[0m"
                changed_coords.append(coord)

        if reset:
            print('\n'.join([''.join(row) for row in self.map]))
            print(flush=True)
            for coord in changed_coords:
                if coord != snc and coord != gnc:
                    self.map[coord[0]][coord[1]] = '.'

    def draw_path_to_image(self, path, final_path=False):
        """ creates images pixel by pixel """
        if final_path:
            adjusted_priorities = [ self.nodes[x].path_length * 255 / self.nodes[self.goal_node_coords].path_length for x in path]
            for i,x in enumerate(path):
                self.pixel_array[x[0]][x[1]] = (0 + adjusted_priorities[i], 0, 255 - adjusted_priorities[i])
        else:
            min_priority = min([x[0] for x in path])
            adjusted_priorities = [x[0]-(min_priority+1) for x in path]
            adjusted_priorities = [x * 255 / max(adjusted_priorities) for x in adjusted_priorities]
            for i,x in enumerate(path):
                self.pixel_array[x[1][0]][x[1][1]] = (255-adjusted_priorities[i], 0, 0+adjusted_priorities[i])

        snc = self.start_node_coords
        gnc = self.goal_node_coords
        self.pixel_array[snc[0]][snc[1]] = (0, 0, 255)
        self.pixel_array[gnc[0]][gnc[1]] = (255, 0, 0)

        pixel_array = np.array(self.pixel_array, np.uint8)
        new_image = Image.fromarray(pixel_array)
        new_image.save(f'example/queue_status_{time.time()}.png')

        if not final_path:
            for i,x in enumerate(path):
                self.pixel_array[x[1][0]][x[1][1]] = (255, 255, 255)

    def create_gif(self):
        """ creates a gif of all the most recently created pngs """
        filenames = [x for x in os.listdir('example') if x.endswith('.png')]
        filenames.sort()
        filenames = filenames[::20] + [filenames[-1] for i in range(240)] #drop frames
        images = list(map(lambda f: imageio.imread('example/'+f), filenames))
        imageio.mimsave(os.path.join('example/a_star_search.gif'), images, format='GIF', fps=120)
        #os.


import libusb1
import usb1
import time
import sys
from struct import *
from datetime import datetime
from mccUSB import *

class bth_1208LS(mccUSB):
  """
    Settings memory map
  |===========================================================================================|
  |    Address    |        Value                                                              |
  |===========================================================================================|
  | 0x000         | DOut Bluetooth connection mode.  This determines the DOut value           |
  |               | when the Bluetooth connection status changes (not applicable in USB mode) |
  |               |    0 = no change                                                          |
  |               |    1 = apply specified values                                             |
  |-------------------------------------------------------------------------------------------|
  | 0x001         | DOut value when Bluetooth connected                                       |
  |-------------------------------------------------------------------------------------------|
  | 0x002         | DOut value when Bluetooth disconnected                                    |
  |-------------------------------------------------------------------------------------------|
  | 0x003         | AOut channel 0 Bluetooth connection mode.  This determines the AOut value |
  |               | when the Bluetooth connection status changes (not applicable in USB mode) |
  |               |    0 = no change                                                          |
  |               |    1 = apply specified values                                             |
  |-------------------------------------------------------------------------------------------|
  | 0x004 - 0x005 | AOut channel 0 value when Bluetooth connected                             |
  |-------------------------------------------------------------------------------------------|
  | 0x006 - 0x007 | AOut channel 0 value when Bluetooth disconnected                          |
  |-------------------------------------------------------------------------------------------|
  | 0x008         | AOut channel 1 Bluetooth connection mode.  This determines the AOut value |
  |               | when the Bluetooth connection status changes (not applicable in USB mode) |
  |               |    0 = no change                                                          |
  |               |    1 = apply specified values                                             |
  |-------------------------------------------------------------------------------------------|
  | 0x009 - 0x00A | AOut channel 1 value when Bluetooth connected                             |
  |-------------------------------------------------------------------------------------------|
  | 0x00B - 0x00C | AOut channel 1 value when Bluetooth disconnected                          |
  |-------------------------------------------------------------------------------------------|
  | 0x00C         | Auto shutdown timer value in minutes (0 = no auto shutdown).  When this   |
  |               | is used the device will automatically power down when powered by          |
  |               | batteries and no Bluetooth connection is present for this amount of time. |
  |-------------------------------------------------------------------------------------------|
  | 0x00E         | Allow charging when Bluetooth connected                                   |
  |               |   0 = do not allow                                                        |
  |               |   1 = allow                                                               |
  |-------------------------------------------------------------------------------------------|
  | 0x00F - 0x3FF | Unused                                                                    |
  |===========================================================================================|
  """

  # Commands and Report ID for BTH-1208LS
  # Digital I/O
  DIN                     = 0x01 # Read the current state of the DIO pins
  DOUT                    = 0x02 # Read/write DIO latch value
  # Analog Input Commands
  AIN                     =  0x10 # Read analog input channel
  AIN_SCAN_START          =  0x11 # Start analog input scan
  AIN_SCAN_STOP           =  0x12 # Stop analog input scan
  AIN_CONFIG              =  0x14 # Read/write analog input configuration
  AIN_SCAN_CLEAR_FIFO     =  0x15 # Clear the analog input scan FIFO
  # Analog Output Commands
  AOUT                    = 0x18  # Read/write analog output channel
  #Counter Commands
  COUNTER                 = 0x20  # Read/reset event counter
  # Memory Commands
  CAL_MEMORY              = 0x30  # Read/write calibration memory
  USER_MEMORY             = 0x31  # Read/write user memory
  SETTINGS_MEMORY         = 0x32  # Read/write settings memory
  # Miscellaneous Commands
  BLINK_LED               = 0x41  # Blink the LED
  RESET                   = 0x42  # Reset the device
  STATUS                  = 0x44  # Read device status
  INIT_RADIO              = 0x45  # Reset radio to default settings
  BLUETOOTH_PIN           = 0x46  # Read/write Bluetooth PIN
  BATTERY_VOLTAGE         = 0x47  # Read battery voltage
  SERIAL                  = 0x48  # Read/write serial number
  RADIO_FIRMWARE_VERSION  = 0x49  # Read radio firmware version
  DFU                     = 0x50  # Enter device firmware upgrade mode

  MAX_PACKET_SIZE         = 64    # max packet size for FS device
  NGAINS                  =  8    # max number of input gain levels (differential mode only)
  NCHAN_DE                =  4    # max number of A/D differential channels
  NCHAN_SE                =  8    # max number of A/D single-ended channels
  NCHAN_AOUT              =  2    # max number of D/A 12 bit 0-2.5V output channels

  # Aanalog Input
  SINGLE_ENDED =  0
  DIFFERENTIAL =  1

  # Analog Input Scan Options
  IMMEDIATE_TRANSFER_MODE = 0x1
  BLOCK_TRANSFER_MODE     = 0x0
  DIFFERENTIAL_MODE       = 0x2
  SINGLE_ENDED_MODE       = 0x0
  NO_TRIGGER              = 0x0
  TRIG_EDGE_RISING        = 0x1 << 2
  TRIG_EDGE_FALLING       = 0x2 << 2
  TRIG_LEVEL_HIGH         = 0x3 << 2
  TRIG_LEVEL_LOW          = 0x4 << 2
  RETRIGGER_MODE          = 0x1 << 5
  STALL_ON_OVERRUN        = 0x0
  INHIBIT_STALL           = 0x1 << 7

  continuous_mode         = False

  # Ranges
  BP_20V   = 0x0   # +/- 20 V
  BP_10V   = 0x1   # +/- 10V
  BP_5V    = 0x2   # +/- 5V
  BP_4V    = 0x3   # +/- 4V
  BP_2_5V  = 0x4   # +/- 2.5V
  BP_2V    = 0x5   # +/- 2V
  BP_1_25V = 0x6   # +/- 1.25V
  BP_1V    = 0x7   # +/- 1V
  UP_2_5V  = 0x8   # 0-2.5V

  # Status bit values
  AIN_SCAN_RUNNING   = 0x1 << 1
  AIN_SCAN_OVERRUN   = 0x1 << 2
  NO_BATTERY         = 0x0
  FAST_CHARGE        = 0x1 << 8
  MAINTENANCE_CHARGE = 0x2 << 8
  FAULT_CHARGING     = 0x3 << 8
  DISABLE_CHARGING   = 0x4 << 8

  def __init__(self, serial=None):
    self.productID = 0x011a
    self.udev = self.openByVendorIDAndProductID(0x9db, self.productID, serial)
    if not self.udev:
      raise IOError("MCC BTH-1208LS not found")

    if sys.platform.startswith('linux'):
      if self.udev.kernelDriverActive(0):
        self.udev.detachKernelDriver(0)
        self.udev.resetDevice()
    # claim all the needed interfaces for AInScan
    self.udev.claimInterface(0)

    # need to get wMaxPacketSize
    self.wMaxPacketSize = self.getMaxPacketSize()

    # Build a lookup table of calibration coefficients to translate values into voltages:
    # The calibration coefficients are stored in the onboard FLASH memory on the device in
    # IEEE-754 4-byte floating point values.
    #
    #   calibrated code = code*slope + intercept
    #
    #  table_AInDE[NCHAN_DE][NGAIN]
    self.table_AInDE  = [[table(), table(), table(), table(), table(), table(), table(), table()],\
                         [table(), table(), table(), table(), table(), table(), table(), table()],\
                         [table(), table(), table(), table(), table(), table(), table(), table()],\
                         [table(), table(), table(), table(), table(), table(), table(), table()]]

    #  table_AInSE = [NCHAN_SE]
    self.table_AInSE = [table(), table(), table(), table(), table(), table(), table(), table()]

    self.BuildGainTable_DE()
    self.BuildGainTable_SE()

  def BuildGainTable_DE(self):
    """
    Builds a lookup table of differential mode calibration
    coefficents to translate values into voltages: The calibration
    coefficients are stored in onboard FLASH memory on the device in
    IEEE-754 4-byte floating point values.

       calibrated code = code * slope + intercept
    """
    address = 0x0
    for gain in range(self.NGAINS):
      for chan in range(self.NCHAN_DE):
        self.table_AInDE[chan][gain].slope, = unpack('f', self.CalMemoryR(address, 4))
        address += 4
        self.table_AInDE[chan][gain].intercept, = unpack('f', self.CalMemoryR(address, 4))
        address += 4

  def BuildGainTable_SE(self):
    """
    Builds a lookup table of single ended multiplexed calibration
    coefficents to translate values into voltages: The calibration
    coefficients are stored in onboard FLASH memory on the device in
    IEEE-754 4-byte floating point values.

       calibrated code = code * slope + intercept
    """

    address = 0x100
    for chan in range(self.NCHAN_SE):
      self.table_AInSE[chan].slope, = unpack('f', self.CalMemoryR(address, 4))
      address += 4
      self.table_AInSE[chan].intercept, = unpack('f', self.CalMemoryR(address, 4))
      address += 4

  def CalDate(self):
    """
    get the manufacturers calibration data (timestamp) from the
    Calibration memory
    """

    # get the year (since 2000)
    address = 0x200
    data ,= unpack('B', self.CalMemoryR(address, 1))
    year  = 2000+data

    # get the month
    address = 0x201
    month ,= unpack('B', self.CalMemoryR(address, 1))

    # get the day
    address = 0x202
    day ,= unpack('B', self.CalMemoryR(address, 1))

    # get the hour
    address = 0x203
    hour ,= unpack('B', self.CalMemoryR(address, 1))

    # get the minute
    address = 0x204
    minute ,= unpack('B', self.CalMemoryR(address, 1))

    # get the second
    address = 0x205
    second ,= unpack('B', self.CalMemoryR(address, 1))

    mdate = datetime(year, month, day, hour, minute, second)
    return mdate

  def volts(self, gain, value):
    """
    converts raw value to volts
    """
    volt = 0.0
    if (gain == self.BP_20V):
      volt = (value - 0x800)* 20.0/2048.
    elif (gain == self.BP_10V):
      volt = (value - 0x800)* 10.0/2048.
    elif (gain == self.BP_5V):
      volt = (value - 0x800)* 5.0/2048.
    elif (gain == self.BP_4V):
      volt = (value - 0x800)* 4.0/2048.
    elif (gain == self.BP_2_5V):
      volt = (value - 0x800)* 2.5/2048.
    elif (gain == self.BP_2V):
      volt = (value - 0x800)* 2.0/2048.
    elif (gain == self.BP_1_25V):
      volt = (value - 0x800)* 1.25/2048.
    elif (gain == self.BP_1V):
      volt = (value - 0x800)* 1.0/2048.
    elif (gain == self.UP_2_5V):
      volt = value*2.5/4096.             # analog output
    else:
      raise ValueError('volts: Unknown range.')

    return volt

  #############################################
  #        Digital I/O Commands               #
  #############################################

  def DIn(self):
    """
    This command reads the current state of the DIO pins.
    """
    request_type = (DEVICE_TO_HOST | VENDOR_TYPE | DEVICE_RECIPIENT)
    wValue = 0
    wIndex = 0
    value ,= unpack('B',self.udev.controlRead(request_type, self.DIN, wValue, wIndex, 1, timeout = 100))
    return value

  def DOut(self, value):
    """
    This command writes the DIO output latch values.  The factory
    power on default is all 1 (pins are floting).  Since the outputs
    are open drain, writing a 0 turns on the low side transistor and
    drives a 0 to the port pin and writing a 1 turns off the
    transistor and allows the pin to float.  The bits are mapped to
    the individual port pins.
    """

    request_type = (HOST_TO_DEVICE | VENDOR_TYPE | DEVICE_RECIPIENT)
    request = self.DOUT
    wValue = value
    wIndex = 0
    result = self.udev.controlWrite(request_type, request, wValue, wIndex, [0x0], timeout = 100)

  def DOutR(self):
    """
    This command reads the DIO output latch values.  The factory
    power on default is all 1 (pins are floting).  Since the outputs
    are open drain, writing a 0 turns on the low side transistor and
    drives a 0 to the port pin and writing a 1 turns off the
    transistor and allows the pin to float.  The bits are mapped to
    the individual port pins.
    """
    request_type = (DEVICE_TO_HOST | VENDOR_TYPE | DEVICE_RECIPIENT)
    wValue = 0
    wIndex = 0
    value ,= unpack('B',self.udev.controlRead(request_type, self.DOUT, wValue, wIndex, 1, timeout = 100))
    return value

  #############################################
  #        Analog Input Commands              #
  #############################################

  def AIn(self, channel, mode, gain):
    """
    This command reads the value of an analog input channel.  This
    command will result in a bus stall if an AInScan is currently
    running.  The range parameter is ignored if the mode is specified
    as single ended.

     channel: the channel to read (0-7)
     mode:    the input mode 0 - single ended, 1 - differential
     range:   the input range for the channel (0-7)
    """
    request_type = (DEVICE_TO_HOST | VENDOR_TYPE | DEVICE_RECIPIENT)
    wValue = (channel | (mode << 0x8))
    wIndex = gain
    value ,= unpack('H',self.udev.controlRead(request_type, self.AIN, wValue, wIndex, 2, timeout = 100))

    if mode == self.SINGLE_ENDED:    # single ended
      data = round(float(value)*self.table_AInSE[channel].slope + self.table_AInSE[channel].intercept)
    else:               # differential
      data = round(float(value)*self.table_AInDE[channel][gain].slope + self.table_AInDE[channel][gain].intercept)
    if (data >= 65536):
      value = 65535
    elif (data < 0):
      value = 0
    else:
      value = data
    return value

  def AInScanStart(self, count, retrig_count, frequency, channels, options):
    """
     This command starts an analog input scan. This command will
    result in a bus stall if an AIn scan is currently running. The
    device will not generate an internal pacer faster than 50 kHz.

    The pacer rate is set by an internal 32-bit timer running at a
    base rate of 40 MHz. The timer is controlled by
    pacer_period. This value is the period of the scan and the A/Ds
    are clocked at this rate. A pulse will be output at the SYNC pin
    at every pacer_period interval if SYNC is configured as an
    output. The equation for calculating pacer_period is:

          pacer_period = [40 MHz / (sample frequency)] - 1

    If pacer_period is set to 0 the device does not generate an A/D
    clock. It uses the SYNC pin as an input and the user must provide
    the pacer source. The A/Ds acquire data on every rising edge of
    SYNC; the maximum allowable input frequency is 50 kHz.  The data
    will be returned in packets utilizing a bulk in endpoint. The
    data will be in the format:

     lowchannel sample 0 : lowchannel + 1 sample 0 : … : hichannel sample 0
     lowchannel sample 1 : lowchannel + 1 sample 1 : … : hichannel sample 1
      …
     lowchannel sample n : lowchannel + 1 sample n : … : hichannel sample n

    The scan will not begin until this command is sent and any
    trigger conditions are met. Data will be sent until reaching the
    specified count or an AInScanStop command is sent.

    The external trigger may be used to start the scan. If enabled,
    the device will wait until the appropriate trigger condition is
    detected then begin sampling data at the specified rate. No
    packets will be sent until the trigger is detected. In retrigger
    mode the trigger will be automatically rearmed and the scan will
    restart after retrig_count samples have been acquired. The count
    parameter specifies the total number of samples to acquire and
    should be >= retrig_count. Specifying 0 for count causes
    continuous retrigger scans. The data is still sent as a
    continuous stream during retrigger scan so the last data from a
    previous scan will not be transferred until the beginning of the
    next retrigger scan if it does not end on a packet boundary.

    In block transfer mode the data is sent in 64-byte packets as
    soon as enough data is available from the A/D. In immediate
    transfer mode the data is sent after each sample period,
    resulting in packets that are always 2 bytes (1 sample.) This
    mode should only be used for low pacer rates, typically under 100
    Hz, because it will overrun much easier.

    Overruns are indicated by the device stalling the bulk in
    endpoint during the scan. The host may read the status to verify
    and must clear the stall condition before further scans can be
    performed.

     count:         the total number of samples to acquire, 0 for continuous scan
     retrig_count:  the number of samples to acquire for each trigger in retrigger mode
     pacer_period:  the pacer timer period (0 for external clock)
     channels:      bit field that selects the channels in the scan, upper 4 bits ignored in differential mode.
        ---------------------------------------------------------
        | Bit7 | Bit6 | Bit5 | Bit4 | Bit3 | Bit2 | Bit1 | Bit0 |
        ---------------------------------------------------------
        | Ch7  |  Ch6 | Ch5  | Ch4  | Ch3  | Ch2  | Ch1  | Ch 0 |
        ---------------------------------------------------------

	options:  bit field that controls scan options
	    bit 0: Reserved
	    bit 1: 1 = differential mode, 0 = single ended mode
	    bits 2-4: Trigger setting:
	      0: no trigger
  	      1: Edge / rising
	      2: Edge / falling
	      3: Level / high
	      4: Level / low
            bit 5: 1 = retrigger mode, 0 = normal trigger mode
	    bit 6: Reserved
	    bit 7: Reserved
    """

    request_type = (HOST_TO_DEVICE | VENDOR_TYPE | DEVICE_RECIPIENT)
    if frequency > 50000.:
      frequency = 50000
    if frequency > 0:
      pacer_period = round((40.E6 / frequency) - 1)
    else:
      pacer_period = 0
    self.frequency = frequency
    self.options = options
    if count == 0:
      self.continuous_mode = True
    else:
      self.continuous_mode = False

    self.nChan = 0
    if options & self.DIFFERENTIAL_MODE:
      for i in range(self.NCHAN_DE):
        if (channels & (0x1 << i)) != 0x0:
          self.nChan += 1
    else:
      for i in range(self.NCHAN_SE):
        if (channels & (0x1 << i)) != 0x0:
          self.nChan += 1

    scanPacket = list(unpack('B'*14, pack('IIIBB', count, retrig_count, pacer_period, channels, options)))
    result = self.udev.controlWrite(request_type, self.AIN_SCAN_START, 0x0, 0x0, scanPacket, timeout = 200)

  def AInScanRead(self, nScan):
    nSamples = int(nScan * self.nChan)

    if self.options & self.IMMEDIATE_TRANSFER_MODE:
      data = []
      for i in range(self.nSamples):
        try:
          data.extend(unpack('H',self.udev.bulkRead(libusb1.LIBUSB_ENDPOINT_IN | 1, 2, 2000)))
        except:
          print('AInScanRead: error in bulk transfer in immmediate transfer mode.')
          return
    else:
      try:
        data =  unpack('H'*nSamples, self.udev.bulkRead(libusb1.LIBUSB_ENDPOINT_IN | 1, int(2*nSamples), 2000))
      except:
        print('AInScanRead: error in bulk transfer!')
        return

    status = self.Status()
    try:
      if status & self.AIN_SCAN_OVERRUN:
        raise OverrunERROR
    except:
      print('AInScanRead: Overrun Error')
      return

    if self.continuous_mode:
      return list(data)

    # if nbytes is a multiple of wMaxPacketSize the device will send a zero byte packet.
    if ((int(nSamples*2) % self.wMaxPacketSize) == 0 and  not(status & self.AIN_SCAN_RUNNING)):
      data2 = self.udev.bulkRead(libusb1.LIBUSB_ENDPOINT_IN | 1, 2, 100)

    self.AInScanStop()
    self.AInScanClearFIFO()
    return list(data)

  def AInScanStop(self):
    """
    This command stops the analog input scan (if running).
    """

    request_type = (HOST_TO_DEVICE | VENDOR_TYPE | DEVICE_RECIPIENT)
    request = self.AIN_SCAN_STOP
    wValue = 0x0
    wIndex = 0x0
    result = self.udev.controlWrite(request_type, request, wValue, wIndex, [0x0], timeout = 100)

  def AInConfigR(self):
    """
    This command reads or writes the analog input ranges used for
    AInScan in differential mode.  The command will result in a bus
    stall if an AInScan is currently running.
    """

    request_type = (DEVICE_TO_HOST | VENDOR_TYPE | DEVICE_RECIPIENT)
    request = self.AIN_CONFIG
    wValue = 0x0
    wIndex = 0x0
    ranges = unpack('BBBB',self.udev.controlRead(request_type, request, wValue, wIndex, 4, timeout = 100))
    return ranges

  def AInConfigW(self, ranges):
    request_type = (HOST_TO_DEVICE | VENDOR_TYPE | DEVICE_RECIPIENT)
    request = self.AIN_CONFIG
    wValue = 0x0
    wIndex = 0x0
    result = self.udev.controlWrite(request_type, request, wValue, wIndex, ranges, timeout = 100)

  def AInScanClearFIFO(self):
    """
    The command clears the internal scan endoint FIFOs
    """
    request_type = (HOST_TO_DEVICE | VENDOR_TYPE | DEVICE_RECIPIENT)
    request = self.AIN_SCAN_CLEAR_FIFO
    wValue = 0x0
    wIndex = 0x0
    result = self.udev.controlWrite(request_type, request, wValue, wIndex, [0x0], timeout = 100)

  #############################################
  #        Analog Output Commands             #
  #############################################

  def AOut(self, channel, value):
    """
    This command writes the values of an analog output channel

      channel: the channel to write (0-1)
      value:   the value to write (0-4095)
    """

    request_type = (HOST_TO_DEVICE | VENDOR_TYPE | DEVICE_RECIPIENT)
    request = self.AOUT
    wValue = value & 0xffff
    wIndex = channel & 0xff

    if channel > 2 or channel < 0:
      raise ValueError('AOut: channel must be 0 or 1.')
      return
    result = self.udev.controlWrite(request_type, request, wValue, wIndex, [0x0], timeout = 100)

  def AOutR(self, channel=0):
    """
    This command reads the value of  both  analog output channels
    """
    request_type = (DEVICE_TO_HOST | VENDOR_TYPE | DEVICE_RECIPIENT)
    request = self.AOUT
    wValue = 0
    wIndex = 0
    channels = unpack('HH',self.udev.controlRead(request_type, request, wValue, wIndex, 4, timeout = 100))
    if channel == 0:
      return channels[0]
    else:
      return channels[1]

  #############################################
  #           Counter Commands                #
  #############################################

  def Counter(self):
    """
    The command reads the event counter.
    """
    request_type = (DEVICE_TO_HOST | VENDOR_TYPE | DEVICE_RECIPIENT)
    wValue = 0
    wIndex = 0
    count ,= unpack('I',self.udev.controlRead(request_type, self.COUNTER, wValue, wIndex, 4, timeout = 100))
    return count

  def ResetCounter(self):
    """
    This command resets the event counter to 0.
    """
    request_type = (HOST_TO_DEVICE | VENDOR_TYPE | DEVICE_RECIPIENT)
    request = self.COUNTER
    wValue = 0
    wIndex = 0
    result = self.udev.controlWrite(request_type, request, wValue, wIndex, [0x0], timeout = 100)

  #############################################
  #        Memory Commands                    #
  #############################################

  def CalMemoryR(self, address, count):
    """
    This command allows for reading the nonvolatile calibration
    memory.  The cal memory is 768 bytes (address 0-0x2FF).  The cal
    memory is write protected and must be unlocked in order to write
    the memory.  The unlock procedure is to write the unlock code
    0xAA55 to address 0x300.  Writes to the entire memory range is
    then possible.  Write any other value to address 0x300 to lock
    the memory after writing.
    """
    request_type = (DEVICE_TO_HOST | VENDOR_TYPE | DEVICE_RECIPIENT)

    if count > 768:
      raise ValueError('CalMemoryR: max bytes that can be read is 768.')
      return

    if address > 0x2ff:
      raise ValueError('CalMemoryR: address must be in the range 0 - 0x2FF.')
      return

    request_type = (DEVICE_TO_HOST | VENDOR_TYPE | DEVICE_RECIPIENT)
    wValue = address & 0xffff   # force to be 16 bits
    wIndex = 0
    try:
      result = self.udev.controlRead(request_type, self.CAL_MEMORY, wValue, wIndex, count, timeout = 100)
    except:
      print("MemoryR: controlRead error")
    return result

  def UserMemoryR(self, address, count):
    """
    This command allow for reading and writing the nonvolatile user
    memory. wLength specifies the number of bytes to read or write.
    The user memory is 256 bytes (address 0-0xFF)
    """
    request_type = (DEVICE_TO_HOST | VENDOR_TYPE | DEVICE_RECIPIENT)
    wValue = address & 0xffff   # force to be 16 bits
    wIndex = 0

    if count > 256:
      raise ValueError('UserMemoryR: max bytes that can be read is 256.')
      return

    if address > 0xff:
      raise ValueError('UserMemoryR: address must be in the range 0 - 0xff.')
      return

    try:
      result = self.udev.controlRead(request_type, self.USER_MEMORY, wValue, wIndex, count, timeout = 100)
    except:
      print("UserMemoryR: controlRead error")

    return result

  def UserMemoyrW(self, address, data):
    request_type = (HOST_TO_DEVICE | VENDOR_TYPE | DEVICE_RECIPIENT)
    wValue = address & 0xffff   # force to be 16 bits
    wIndex = 0

    if count > 256:
      raise ValueError('UserMemoryW: max bytes that can be read is 256.')
      return

    if address > 0xff:
      raise ValueError('UserMemoryW: address must be in the range 0 - 0xff.')
      return

    try:
      result = self.udev.controlWrite(request_type, self.USER_MEMORY, wValue, wIndex, data, timeout = 100)
    except:
      print("UserMemoryW: controlWrite error")

  def SettingsMemoryR(self, address, count):
    """
    This command allows for reading and writing the nonvolite
    settings memory.  wLength specifies the number of bytes to read
    or write.  The settings memory is 1024 bytes (address 0 - 0x3FF).  If
    the settings are written they will be implemented immediately.

    This command allow for reading and writing the nonvolatile user
    memory. wLength specifies the number of bytes to read or write.
    The user memory is 256 bytes (address 0-0xFF)
    """

    request_type = (DEVICE_TO_HOST | VENDOR_TYPE | DEVICE_RECIPIENT)
    wValue = address & 0xffff   # force to be 16 bits
    wIndex = 0

    if count > 512:
      raise ValueError('SettingsMemoryR: max bytes that can be read is 512.')
      return

    if address > 0x3ff:
      raise ValueError('SettingsMemoryR: address must be in the range 0 - 0x3ff.')
      return

    try:
      result = self.udev.controlRead(request_type, self.SETTINGS_MEMORY, wValue, wIndex, count, timeout = 100)
    except:
      print("SettingsMemoryR: controlRead error")

    return result

  def SettingsMemoyrW(self, address, data):
    request_type = (HOST_TO_DEVICE | VENDOR_TYPE | DEVICE_RECIPIENT)
    wValue = address & 0xffff   # force to be 16 bits
    wIndex = 0

    if count > 512:
      raise ValueError('SettingsMemoryW: max bytes that can be read is 512.')
      return

    if address > 0x3ff:
      raise ValueError('SettingsMemoryW: address must be in the range 0 - 0x3ff.')
      return

    try:
      result = self.udev.controlWrite(request_type, self.SETTINGS_MEMORY, wValue, wIndex, data, timeout = 100)
    except:
      print("SettingsMemoryW: controlWrite error")

  #############################################
  #        Miscellaneous Commands             #
  #############################################

  def BlinkLED(self, count):
    """
    This command will blink the device LED
    """
    request_type = (HOST_TO_DEVICE | VENDOR_TYPE | DEVICE_RECIPIENT)
    request = self.BLINK_LED
    wValue = 0
    wIndex = 0
    result = self.udev.controlWrite(request_type, request, wValue, wIndex, [count], timeout = 100)

  def Reset(self):
    """
    This command resets the device
    """
    request_type = (HOST_TO_DEVICE | VENDOR_TYPE | DEVICE_RECIPIENT)
    request = self.RESET
    wValue = 0
    wIndex = 0
    result = self.udev.controlWrite(request_type, request, wValue, wIndex, [0x0], timeout = 100)

  def Status(self):
    """
    This command retrieves the status of the device.  Writing the command will clear
    the error indicators.

    status:      bit 0:      Reserved
                 bit 1:      1 = AIn scan running
                 bit 2:      1 = AIn scan overrun
                 bits 3-7:   Reserved
                 bits 8-10:  Charger status:
                              0 = no battery
                              1 = fast charge
                              2 = maintenance charge
                              3 = fault (not charging)
                              4 = disabled
                 bits 11-15   Reserved
    """
    request_type = (DEVICE_TO_HOST | VENDOR_TYPE | DEVICE_RECIPIENT)
    wValue = 0
    wIndex = 0
    status ,= unpack('H',self.udev.controlRead(request_type, self.STATUS, wValue, wIndex, 2, timeout = 100))
    return status

  def InitRadio(self):
    """
    This command sets the radio module to factory default settings.
    This should be used during manufacturing to initialize the radio.
    """
    request_type = (HOST_TO_DEVICE | VENDOR_TYPE | DEVICE_RECIPIENT)
    request = self.INIT_RADIO
    wValue = 0
    wIndex = 0
    result = self.udev.controlWrite(request_type, request, wValue, wIndex, [0x0], timeout = 100)

  def BluetoothPinR(self):
    """
    This command reads the Bluetooth PIN code.  The PIN code can be up
    to 16 ASCII characters.  The length of the code is specified in
    wLength.
    """
    request_type = (DEVICE_TO_HOST | VENDOR_TYPE | DEVICE_RECIPIENT)
    wValue = 0
    wIndex = 0
    pin = self.udev.controlRead(request_type, self.BLUETOOTH_PIN, wValue, wIndex, 16, timeout = 100)
    return  pin.decode()

  def BluetoothPinW(self, pin):
    """
    This command writes the Bluetooth PIN code.  The PIN code can be up
    to 16 ASCII characters.  The length of the code is specified in
    wLength.
    """
    if len(pin) > 16:
      raise ValueError('BluetoothPinW: length of pin greater than 16.')
      return

    request_type = (HOST_TO_DEVICE | VENDOR_TYPE | DEVICE_RECIPIENT)
    wValue = 0
    wIndex = 0
    self.udev.controlWrite(request_type, self.BLUETOOTH_PIN, wValue, wIndex, [pin], timeout = 100)

  def BatteryVoltage(self):
    """
    This command reads the voltage of the battery.  The voltage is
    returned in millivolts.
    """
    request_type = (DEVICE_TO_HOST | VENDOR_TYPE | DEVICE_RECIPIENT)
    wValue = 0
    wIndex = 0
    voltage ,= unpack('H',self.udev.controlRead(request_type, self.BATTERY_VOLTAGE, wValue, wIndex, 2, timeout = 100))
    return voltage

  def RadioFirmwareVersion(self):
    """
    This command reads the radio firmware version.  The version
    consists of 16 bits in hexadecimal BCD notation.  i.e. version
    6.15 would be 0x0615.
    """
    request_type = (DEVICE_TO_HOST | VENDOR_TYPE | DEVICE_RECIPIENT)
    wValue = 0
    wIndex = 0
    version ,= unpack('H',self.udev.controlRead(request_type, self.RADIO_FIRMWARE_VERSION, wValue, wIndex, 2, timeout = 100))
    return version

  def DFU(self):
    """
    This command places the device in firmware upgrade mode by erasing
    a portion of the program memory.  The next time the device is
    reset, it will enumerate in the bootloader and is unusable as a
    DAQ device until new firmware is loaded.
    """
    request_type = (HOST_TO_DEVICE | VENDOR_TYPE | DEVICE_RECIPIENT)
    wValue = 0xADAD
    wIndex = 0
    self.udev.controlWrite(request_type, self.DFU, wValue, wIndex, [0x0], timeout = 100)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''
Created on Aug 5, 2016

@author: finetu
'''

from bs4 import BeautifulSoup
import hashlib
import json
import logging
import re
from scrapy.selector import Selector
from six import string_types
import xmltodict
import zorba_api

from web_crawler.items import WebTargetRes
from web_crawler.scrapyredis.spiders import RedisSpider

logger = logging.getLogger(__name__)
class WebSpider(RedisSpider):
    """Spider that reads urls from redis queue (myspider:start_urls)."""
    handle_httpstatus_list = [302,403,404,500]
    name = 'web_spider'
    redis_key = 'web_spider:url_queue'
    zorbaStore = zorba_api.InMemoryStore_getInstance()
    zorba = zorba_api.Zorba_getInstance(zorbaStore)
    hrefRex = re.compile(r'(https?://.*?/)')
    imgRex = re.compile(r'.+(jpg|JPG|png|PNG)$')
    image_urls = []
    httpDomain = ""
    def __init__(self, *args, **kwargs):

        super(WebSpider, self).__init__(*args, **kwargs)

    def parse(self, response):
        try:
            self.image_urls = []
            self.httpDomain = ""
            if response.status != 200 :
                logger.info(response.url+" crawl failure! status:"+str(response.status))
                return
            if (not response.meta.has_key('xpath')) :
                logger.info(response.url+"can not find xpath in response.meta: "+str(response.meta))
                return
            webTargetRes = WebTargetRes()

            webTargetRes["meta"] = {}
            webTargetRes["meta"]["url"] = response.url
            self.httpDomain = self.hrefRex.search(response.url).group(1)
            if response.meta.has_key('target_id'):
                webTargetRes["meta"]["target_id"] = response.meta['target_id']
            if response.meta.has_key('try_key'):
                webTargetRes["meta"]["try_key"] = response.meta['try_key']

            if response.meta.has_key('xquery'):
                sel = Selector(response)
                resBody = sel.xpath(response.meta['xpath']).extract()
                if len(resBody) == 0:
                    logger.info(response.url+",xpath:"+response.meta['xpath']+" no elements found on this xpath.")
                    return
                resBody = resBody[0]
                xquery = response.meta['xquery']
                jsonObj = self.genJsonByXquery(resBody.encode("utf8"),xquery.encode("utf8"))
                webTargetRes["image_urls"] = self.image_urls
                webTargetRes['data'] = jsonObj['data']
                jsonStr = json.dumps(jsonObj)
                md5 = hashlib.md5(jsonStr).hexdigest()
                webTargetRes["meta"]["md5"] = md5
            else:
                sel = Selector(response)
                news_list = sel.xpath(response.meta['xpath']+'//a')

                items = []
                for news in news_list:
                    item = {}
                    titleStr = ""
                    names =  news.xpath('text()').extract()
                    if len(names) == 0:
                        continue
                    title = news.xpath('text()').extract()[0]
                    link = news.xpath('@href').extract()[0]
                    item['text'] = title.strip()
                    item['href'] = link.strip()
                    titleStr += title.strip()
                    items.append(item)

                webTargetRes["data"] = items
                md5 = hashlib.md5(titleStr).hexdigest()
                webTargetRes["meta"]["md5"] = md5
                self.validate(webTargetRes)

            webTargetRes['meta']['is_success'] = True
            yield webTargetRes
        except Exception as e:
            logger.error("Exception in parse: %s", str(e))
            meta = response.meta
            webTargetRes = WebTargetRes()
            webTargetRes["meta"] = {}
            webTargetRes["meta"]["url"] = meta["url"]
            if meta.has_key('target_id'):
                webTargetRes["meta"]["target_id"] = meta['target_id']
            if meta.has_key('try_key'):
                webTargetRes["meta"]["try_key"] = meta['try_key']
            webTargetRes["meta"]["is_success"] = False
            webTargetRes["data"] = {"msg":str(e)}

            yield webTargetRes

    def genJsonByXquery(self,resBody,xquery):
        soup = BeautifulSoup(resBody,"lxml")
        prettyStr = soup.prettify().encode("utf8")

        dataManager = self.zorba.getXmlDataManager()
        docIter = dataManager.parseXML(prettyStr)
        docIter.open();

        doc = zorba_api.Item_createEmptyItem()
        docIter.next(doc)
        docIter.close()
        docIter.destroy()

        try:
            docManager = dataManager.getDocumentManager()
            docManager.put("response", doc)
            xquery = self.zorba.compileQuery(xquery)
            dynCtx = xquery.getDynamicContext();
            dynCtx.setContextItem(doc);
            xmlStr = xquery.execute()
        finally:
            docManager.remove("response")
        xmlStr = xmlStr.replace("<?xml version=\"1.0\" encoding=\"UTF-8\"?>","").strip()
        xmlStr = "<data>"+xmlStr+"</data>"
        jsonObj = self.toJsonObj(xmlStr)["data"]
        self.append_image_urls(jsonObj)
        return jsonObj

    def toJsonObj(self,xmlStr):
        jsonStr = json.dumps(xmltodict.parse(xmlStr))
        return json.loads(jsonStr)

    def validate(self,webTargetRes):
        prefix = self.hrefRex.search(webTargetRes["meta"]['url']).group(1)
#         linkMap = {}

        for i in webTargetRes["data"]:
            if i['text']==None or i['text'].strip()=="" or i['href']==None or i['href'].strip()=="" :
                continue
            i["text"] = i["text"].encode("utf8")
            i["href"] = i["href"].encode("utf8")
            if i['href'].find('http://') == -1 :
                i['href'] = prefix + i['href']

#             if not linkMap.has_key(i['href']) or len(linkMap[i['href']]) < len(i['text']):
#                 linkMap[i['href']] = i['text'].replace('\"','\\\"').strip()
#
#         jsonObj = []
#         for i in items:
#             if not linkMap.has_key(i["href"]):
#                 continue
#             jsonObj.append({"href":i["href"],"text":linkMap[i['href']]})
#             linkMap[i['href']] = None
#
#         jsonStr = json.dumps(jsonObj)
#         return webTargetRes

#     def parse(self, response):
#         if response.status != 200 :
#             logger.info(response.url+" crawl failure! status:"+str(response.status))
#         if (not response.meta.has_key('xpath')) :
#             logger.info(response.url+"can not find xpath in response.meta: "+str(response.meta))
#         webTargetRes = WebTargetRes()
#
#         webTargetRes["target"] = {}
#         webTargetRes["target"]["url"] = response.url
#         if response.meta.has_key('target_id'):
#             webTargetRes["target"]["id"] = response.meta['target_id']
#         if response.meta.has_key('tryKey'):
#             webTargetRes["tryKey"] = response.meta['tryKey']
#         sel = Selector(response)
#         news_list = sel.xpath(response.meta['xpath']+'//a')
#
#         items = []
#         for news in news_list:
#             item = {}
#             titleStr = ""
#             names =  news.xpath('text()').extract()
#             if len(names) == 0:
#                 continue
#             title = news.xpath('text()').extract()[0]
#             link = news.xpath('@href').extract()[0]
#             item['text'] = title.strip()
#             item['href'] = link.strip()
#             titleStr += title.strip()
#             logger.info(title.encode('utf-8'))
#             items.append(item)
#
#         webTargetRes["linkList"] = items
#         md5 = hashlib.md5(titleStr).hexdigest()
#         webTargetRes["md5"] = md5
#         yield webTargetRes

    def append_image_urls(self,jsonObj):
        if not isinstance(jsonObj,dict) and not isinstance(jsonObj,list):
            return
        if isinstance(jsonObj,dict):
            for entry in jsonObj.iteritems():
                key = entry[0]
                value = entry[1]
                if isinstance(value,string_types) and self.imgRex.search(value)!=None:
                    oldUrl = self.convert_url(value,self.httpDomain)
                    self.image_urls.append(oldUrl)
#                     hascode = hashlib.sha1(self.to_bytes(oldUrl)).hexdigest()
#                     newUrl = IMAGES_URL_PREX+hascode+".jpg"
                    jsonObj[key] = oldUrl
                if isinstance(value,dict) or isinstance(value,list):
                    self.append_image_urls(value)
        if isinstance(jsonObj,list):
            for index in range(len(jsonObj)):
                value = jsonObj[index]
                if isinstance(value,string_types) and self.imgRex.search(value)!=None:
                    oldUrl = self.convert_url(value,self.httpDomain)
                    self.image_urls.append(oldUrl)
#                     hascode = hashlib.sha1(self.to_bytes(oldUrl)).hexdigest()
#                     newUrl = IMAGES_URL_PREX+hascode+".jpg"
                    jsonObj[index] = oldUrl
                if isinstance(value,dict) or isinstance(value,list):
                    self.append_image_urls(value)

#     def to_bytes(self,text, encoding=None, errors='strict'):
#         if isinstance(text, bytes):
#             return text
#         if not isinstance(text, string_types):
#             raise TypeError('to_bytes must receive a unicode, str or bytes '
#                             'object, got %s' % type(text).__name__)
#         if encoding is None:
#             encoding = 'utf-8'
#         return text.encode(encoding, errors)

    def convert_url(self,url,httpDomain):
        if url.startswith("http"):
            return url
        elif url.startswith("//"):
            return "http:"+url
        else:
            return httpDomain+url

    def make_request_from_data(self, data):
        if not data.has_key("url"):
            logger.error("Unexpected URL from '%s': %r", self.redis_key, data)
        url = data['url']
        if '://' not in url:
            logger.error("Unexpected URL from '%s': %r", self.redis_key, data)

        request = self.make_requests_from_url(url)
#         data.pop("url")
        request.meta.update(data)
        request.replace(dont_filter=True)
        request.callback = self.parse
        request.errback = lambda err: self.handle_request_failure(err, request.meta)
        return request

    def handle_request_failure(self,err,meta):
        logger.warning("download failed: %s ", str(err))

        webTargetRes = WebTargetRes()
        webTargetRes["meta"] = {}
        webTargetRes["meta"]["url"] = meta["url"]
        if meta.has_key('target_id'):
            webTargetRes["meta"]["target_id"] = meta['target_id']
        if meta.has_key('try_key'):
            webTargetRes["meta"]["try_key"] = meta['try_key']
        webTargetRes["meta"]["is_success"] = False
        webTargetRes["data"] = {"msg":str(err).strip()}

        yield webTargetRes


#-*-coding:utf-8 -*-
__author__ = 'Jy2881'

import numpy as np
import matplotlib.pyplot as plt
import multivariateGaussian as mG

def visualizeFit(X,  mu, sigma2):
    X1,X2 = np.meshgrid(np.arange(0,35,0.5),np.arange(0,35,0.5))
    m,n = np.shape(X1)
    a = X1.ravel().reshape(m*n,1)

    Z = mG.multivariateGaussian(np.c_[(X1.ravel().reshape(m*n,1),X2.ravel().reshape(m*n,1))],mu,sigma2)
    Z = Z.reshape(np.shape(X1))

    # plot points in buttom layer
    plt.scatter(X[:,0],X[:,1],marker="x",color='b',linewidths=0.01)
    plt.axis([0,30,0,30])
    plt.xlabel('Latency (ms)')
    plt.ylabel('Throughput (mb/s)')

    # plot countours; Do not plot if there are infinities
    if (np.sum(np.isinf(Z)) == 0):
        C = plt.contour(X1, X2, Z, 10, colors = 'black', linewidth = 0.5)
        # C.contour(X1, X2, Z, 10**np.arange(-20,0,3))
    plt.xlabel('Latency (ms)')
    plt.ylabel('Throughput (mb/s)')
    plt.show()


import numpy as np # linear algebra

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

df = pd.read_csv('../input/train.csv')

df.dropna(axis=0)

df.set_index('id', inplace = True)

df.head()
import re

from nltk.corpus import stopwords

stopWords = set(stopwords.words('english'))

#creating a function to encapsulate preprocessing, to mkae it easy to replicate on  submission data

def processing(df):

    #lowering and removing punctuation

    df['processed'] = df['text'].apply(lambda x: re.sub(r'[^\w\s]','', x.lower()))

    #numerical feature engineering

    #total length of sentence

    df['length'] = df['processed'].apply(lambda x: len(x))

    #get number of words

    df['words'] = df['processed'].apply(lambda x: len(x.split(' ')))

    df['words_not_stopword'] = df['processed'].apply(lambda x: len([t for t in x.split(' ') if t not in stopWords]))

    #get the average word length

    df['avg_word_length'] = df['processed'].apply(lambda x: np.mean([len(t) for t in x.split(' ') if t not in stopWords]) if len([len(t) for t in x.split(' ') if t not in stopWords]) > 0 else 0)

    #get the average word length

    df['commas'] = df['text'].apply(lambda x: x.count(','))

    return(df)

df = processing(df)

df.head()
from sklearn.model_selection import train_test_split

features= [c for c in df.columns.values if c  not in ['id','text','author']]

numeric_features= [c for c in df.columns.values if c  not in ['id','text','author','processed']]

target = 'author'

X_train, X_test, y_train, y_test = train_test_split(df[features], df[target], test_size=0.33, random_state=42)

print('X_train size: {}, X_test size: {}'.format(X_train.shape, X_test.shape))

X_train.head()
from sklearn.base import BaseEstimator, TransformerMixin

class DataFrameSelector(BaseEstimator, TransformerMixin):

    """

     Transformer to select a part of the data frame to perform additional transformations on

     """

    def __init__(self, attribute_names):

        self.attribute_names = attribute_names

    def fit(self, X, y = None):

        return self

    def transform(self, X):

        return X[self.attribute_names]

str_feature = 'processed'

num_features_list = ['length', 'words', 'words_not_stopword', 'avg_word_length', 'commas']
from sklearn.pipeline import Pipeline

from sklearn.feature_extraction.text import TfidfVectorizer

text = Pipeline([

                ('selector', DataFrameSelector(str_feature)),

                ('tfidf', TfidfVectorizer( stop_words='english'))

            ])

text.fit_transform(X_train)
from sklearn.preprocessing import StandardScaler

num_feats = Pipeline([

    ('selector', DataFrameSelector(num_features_list)),

    ('standard', StandardScaler())

])

num_feats.fit_transform(X_train)
from sklearn.pipeline import FeatureUnion

feats = FeatureUnion([

    ('text', text),

    ('num_feats', num_feats)

])

feature_processing = Pipeline([('feats', feats)])

feature_processing.fit_transform(X_train)
from sklearn.ensemble import RandomForestClassifier

pipeline = Pipeline([

    ('features',feats),

    ('classifier', RandomForestClassifier(random_state = 42)),

])

pipeline.fit(X_train, y_train)

preds = pipeline.predict(X_test)

np.mean(preds == y_test)
pipeline.get_params().keys()
from sklearn.model_selection import GridSearchCV

hyperparameters = { 'features__text__tfidf__max_df': [0.9, 0.95],

                    'features__text__tfidf__ngram_range': [(1,1), (1,2)],

                   'classifier__max_depth': [50, 70],

                    'classifier__min_samples_leaf': [1,2]

                  }

clf = GridSearchCV(pipeline, hyperparameters, cv=5)

# Fit and tune model

clf.fit(X_train, y_train)
clf.best_params_
#refitting on entire training data using best settings

clf.refit

preds = clf.predict(X_test)

probs = clf.predict_proba(X_test)

np.mean(preds == y_test)
submission = pd.read_csv('../input/test.csv')

#preprocessing

submission = processing(submission)

predictions = clf.predict_proba(submission)

preds = pd.DataFrame(data=predictions, columns = clf.best_estimator_.named_steps['classifier'].classes_)

#generating a submission file

result = pd.concat([submission[['id']], preds], axis=1)

result.set_index('id', inplace = True)

result.head()