# -*-coding: utf-8 -*-

from __future__ import absolute_import, division, print_function

import os
import tensorflow as tf
import tensorflow.contrib.slim as slim
import numpy as np

from libs.networks import resnet
from libs.networks import resnet_gluoncv
from libs.networks import mobilenet_v2
from libs.box_utils import encode_and_decode
from libs.box_utils import anchor_utils
from libs.losses import losses
from libs.box_utils import show_box_in_tensor
from libs.detection_oprations.proposal_opr import postprocess_rpn_proposals
from libs.detection_oprations.anchor_target_layer_without_boxweight import anchor_target_layer
from libs.detection_oprations.proposal_target_layer import proposal_target_layer
from libs.box_utils import mask_utils
from libs.label_name_dict.label_dict import *
from libs.box_utils import boxes_utils

class DetectionNetwork(object):
    def __init__(self, base_network_name, is_training):

        self.base_network_name = base_network_name
        self.is_training = is_training
        self.num_anchors_per_location = len(cfgs.ANCHOR_SCALES) * len(cfgs.ANCHOR_RATIOS)
        self.loss_dict = {}

    def build_base_network(self, input_img_batch):

        if self.base_network_name.startswith('resnet_v1'):
            return resnet.resnet_base(input_img_batch, scope_name=self.base_network_name, is_training=self.is_training)
        elif self.base_network_name.startswith('resnet'):
            print("use resnet_from_Mxnet")
            return resnet_gluoncv.resnet_base(input_img_batch, scope_name=self.base_network_name,
                                              is_training=self.is_training)
        elif self.base_network_name.startswith('MobilenetV2'):
            return mobilenet_v2.mobilenetv2_base(input_img_batch, is_training=self.is_training)

        else:
            raise ValueError('Sry, we only support resnet or mobilenet_v2')

    def postprocess_fastrcnn(self, rois, bbox_ppred, scores, img_shape):
        '''
        :param rois:[-1, 4]
        :param bbox_ppred: [-1, (cfgs.Class_num+1) * 4]
        :param scores: [-1, cfgs.Class_num + 1]
        :return:
        '''

        with tf.name_scope('postprocess_fastrcnn'):
            rois = tf.stop_gradient(rois)
            scores = tf.stop_gradient(scores)
            bbox_ppred = tf.reshape(bbox_ppred, [-1, cfgs.CLASS_NUM + 1, 4])
            bbox_ppred = tf.stop_gradient(bbox_ppred)

            bbox_pred_list = tf.unstack(bbox_ppred, axis=1)
            score_list = tf.unstack(scores, axis=1)

            allclasses_boxes = []
            allclasses_scores = []
            categories = []
            for i in range(1, cfgs.CLASS_NUM+1):

                # 1. decode boxes in each class
                tmp_encoded_box = bbox_pred_list[i]
                tmp_score = score_list[i]
                tmp_decoded_boxes = encode_and_decode.decode_boxes(encoded_boxes=tmp_encoded_box,
                                                                   reference_boxes=rois,
                                                                   scale_factors=cfgs.ROI_SCALE_FACTORS)

                # 2. clip to img boundaries
                tmp_decoded_boxes = boxes_utils.clip_boxes_to_img_boundaries(decode_boxes=tmp_decoded_boxes,
                                                                             img_shape=img_shape)

                # 3. NMS
                if cfgs.SOFT_NMS:
                    print("Using Soft NMS.......")
                    raise NotImplementedError("soft NMS has not implemented")

                else:
                    keep = tf.image.non_max_suppression(
                        boxes=tmp_decoded_boxes,
                        scores=tmp_score,
                        max_output_size=cfgs.FAST_RCNN_NMS_MAX_BOXES_PER_CLASS,
                        iou_threshold=cfgs.FAST_RCNN_NMS_IOU_THRESHOLD)

                perclass_boxes = tf.gather(tmp_decoded_boxes, keep)
                perclass_scores = tf.gather(tmp_score, keep)

                allclasses_boxes.append(perclass_boxes)
                allclasses_scores.append(perclass_scores)
                categories.append(tf.ones_like(perclass_scores) * i)

            final_boxes = tf.concat(allclasses_boxes, axis=0)
            final_scores = tf.concat(allclasses_scores, axis=0)
            final_category = tf.concat(categories, axis=0)

            if self.is_training:
                '''
                in training. We should show the detecitons in the tensorboard. So we add this.
                '''
                kept_indices = tf.reshape(tf.where(tf.greater_equal(final_scores, cfgs.SHOW_SCORE_THRSHOLD)), [-1])
                final_boxes = tf.gather(final_boxes, kept_indices)
                final_scores = tf.gather(final_scores, kept_indices)
                final_category = tf.gather(final_category, kept_indices)

            return final_boxes, final_scores, final_category

    def roi_pooling(self, feature_maps, rois, img_shape, scope):
        '''
        Here use roi warping as roi_pooling

        :param featuremaps_dict: feature map to crop
        :param rois: shape is [-1, 4]. [x1, y1, x2, y2]
        :return:
        '''

        with tf.variable_scope('ROI_Warping_' + scope):
            img_h, img_w = tf.cast(img_shape[1], tf.float32), tf.cast(img_shape[2], tf.float32)
            N = tf.shape(rois)[0]
            x1, y1, x2, y2 = tf.unstack(rois, axis=1)

            normalized_x1 = x1 / img_w
            normalized_x2 = x2 / img_w
            normalized_y1 = y1 / img_h
            normalized_y2 = y2 / img_h

            normalized_rois = tf.transpose(
                tf.stack([normalized_y1, normalized_x1, normalized_y2, normalized_x2]), name='get_normalized_rois')

            normalized_rois = tf.stop_gradient(normalized_rois)

            cropped_roi_features = tf.image.crop_and_resize(feature_maps, normalized_rois,
                                                            box_ind=tf.zeros(shape=[N, ],
                                                                             dtype=tf.int32),
                                                            crop_size=[cfgs.ROI_SIZE, cfgs.ROI_SIZE],
                                                            name='CROP_AND_RESIZE'
                                                            )
            roi_features = slim.max_pool2d(cropped_roi_features,
                                           [cfgs.ROI_POOL_KERNEL_SIZE, cfgs.ROI_POOL_KERNEL_SIZE],
                                           stride=cfgs.ROI_POOL_KERNEL_SIZE)

        return roi_features

    def build_fastrcnn(self, P_list, rois_list, img_shape):

        with tf.variable_scope('Fast-RCNN'):
            # 5. ROI Pooling
            with tf.variable_scope('rois_pooling'):
                pooled_features_list = []
                for level_name, p, rois in zip(cfgs.LEVLES, P_list, rois_list):  # exclude P6_rois
                    # p = tf.Print(p, [tf.shape(p)], summarize=10, message=level_name+'SHPAE***')
                    pooled_features = self.roi_pooling(feature_maps=p, rois=rois, img_shape=img_shape,
                                                       scope=level_name)
                    pooled_features_list.append(pooled_features)

                pooled_features = tf.concat(pooled_features_list, axis=0)  # [minibatch_size, H, W, C]

            # 6. inferecne rois in Fast-RCNN to obtain fc_flatten features
            if self.base_network_name.startswith('resnet'):
                fc_flatten = resnet.restnet_head(inputs=pooled_features,
                                                 is_training=self.is_training,
                                                 scope_name=self.base_network_name)
            elif self.base_network_name.startswith('Mobile'):
                fc_flatten = mobilenet_v2.mobilenetv2_head(inputs=pooled_features,
                                                           is_training=self.is_training)
            else:
                raise NotImplementedError('only support resnet and mobilenet')

            # 7. cls and reg in Fast-RCNN
            with slim.arg_scope([slim.fully_connected], weights_regularizer=slim.l2_regularizer(cfgs.WEIGHT_DECAY)):

                cls_score = slim.fully_connected(fc_flatten,
                                                 num_outputs=cfgs.CLASS_NUM + 1,
                                                 weights_initializer=cfgs.INITIALIZER,
                                                 activation_fn=None, trainable=self.is_training,
                                                 scope='cls_fc')
                # if cfgs.ADD_EXTR_CONVS_FOR_REG > 0:
                #     bbox_input_feat = pooled_features
                #     for i in range(cfgs.ADD_EXTR_CONVS_FOR_REG):
                #         bbox_input_feat = slim.conv2d(bbox_input_feat, num_outputs=256, kernel_size=[3, 3], stride=1,
                #                                       padding="SAME", scope='extra_conv%d' % i)
                #     bbox_input_fc_feat = slim.flatten(bbox_input_feat, scope='bbox_feat_flatten')
                #
                #     bbox_pred = slim.fully_connected(bbox_input_fc_feat,
                #                                      num_outputs=(cfgs.CLASS_NUM + 1) * 4,
                #                                      weights_initializer=cfgs.BBOX_INITIALIZER,
                #                                      activation_fn=None, trainable=self.is_training,
                #                                      scope='reg_fc')
                # else:
                #     bbox_pred = slim.fully_connected(fc_flatten,
                #                                      num_outputs=(cfgs.CLASS_NUM + 1) * 4,
                #                                      weights_initializer=cfgs.BBOX_INITIALIZER,
                #                                      activation_fn=None, trainable=self.is_training,
                #                                      scope='reg_fc')

                bbox_input_feat = pooled_features
                for i in range(cfgs.ADD_EXTR_CONVS_FOR_REG):
                    bbox_input_feat = slim.conv2d(bbox_input_feat, num_outputs=256, kernel_size=[3, 3], stride=1,
                                                  padding="SAME", scope='extra_conv%d' % i)
                bbox_input_fc_feat = slim.flatten(bbox_input_feat, scope='bbox_feat_flatten')

                bbox_pred = slim.fully_connected(bbox_input_fc_feat,
                                                 num_outputs=(cfgs.CLASS_NUM + 1) * 4,
                                                 weights_initializer=cfgs.BBOX_INITIALIZER,
                                                 activation_fn=None, trainable=self.is_training,
                                                 scope='reg_fc')

                cls_score = tf.reshape(cls_score, [-1, cfgs.CLASS_NUM + 1])
                bbox_pred = tf.reshape(bbox_pred, [-1, 4 * (cfgs.CLASS_NUM + 1)])

                return bbox_pred, cls_score

    def build_concat_fastrcnn(self, P_list, all_rois, img_shape):

        with tf.variable_scope('Fast-RCNN'):

            # 5.0 concat feature maps:

            h, w = tf.shape(P_list[0])[1], tf.shape(P_list[0])[2]

            concat_list = [P_list[0]]
            for l in range(1, len(cfgs.LEVLES)):  # do not concat P6, do not upsample P2
                upsample_p = tf.image.resize_bilinear(P_list[l],
                                                      size=[h, w],
                                                      name='up_sample_%d' % (l + 2))
                concat_list.append(upsample_p)
            concat_fet = tf.concat(concat_list, axis=-1)
            if cfgs.CONCAT_CHANNEL != 1024:
                print("concat channel is not 1024")
                concat_fet = slim.conv2d(concat_fet, activation_fn=None,
                                         num_outputs=cfgs.CONCAT_CHANNEL, kernel_size=[3, 3], padding="SAME",
                                         stride=1, scope="concat_fets", biases_initializer=None)

            # 5.1. ROI Pooling
            with tf.variable_scope('rois_pooling'):

                pooled_features = self.roi_pooling(concat_fet, rois=all_rois,
                                                   img_shape=img_shape, scope="all")

            # 6. inferecne rois in Fast-RCNN to obtain fc_flatten features
            if self.base_network_name.startswith('resnet'):
                fc_flatten = resnet.restnet_head(inputs=pooled_features,
                                                 is_training=self.is_training,
                                                 scope_name=self.base_network_name)
            elif self.base_network_name.startswith('Mobile'):
                fc_flatten = mobilenet_v2.mobilenetv2_head(inputs=pooled_features,
                                                           is_training=self.is_training)
            else:
                raise NotImplementedError('only support resnet and mobilenet')

            # 7. cls and reg in Fast-RCNN
            with slim.arg_scope([slim.fully_connected], weights_regularizer=slim.l2_regularizer(cfgs.WEIGHT_DECAY)):

                cls_score = slim.fully_connected(fc_flatten,
                                                 num_outputs=cfgs.CLASS_NUM + 1,
                                                 weights_initializer=cfgs.INITIALIZER,
                                                 activation_fn=None, trainable=self.is_training,
                                                 scope='cls_fc')
                if cfgs.ADD_EXTR_CONVS_FOR_REG > 0:
                    bbox_input_feat = pooled_features
                    for i in range(cfgs.ADD_EXTR_CONVS_FOR_REG):
                        bbox_input_feat = slim.conv2d(bbox_input_feat, num_outputs=256, kernel_size=[3, 3], stride=1,
                                                      padding="SAME", scope='extra_conv%d' % i)
                    bbox_input_fc_feat = slim.flatten(bbox_input_feat, scope='bbox_feat_flatten')

                    bbox_pred = slim.fully_connected(bbox_input_fc_feat,
                                                     num_outputs=(cfgs.CLASS_NUM + 1) * 4,
                                                     weights_initializer=cfgs.BBOX_INITIALIZER,
                                                     activation_fn=None, trainable=self.is_training,
                                                     scope='reg_fc')
                else:
                    bbox_pred = slim.fully_connected(fc_flatten,
                                                     num_outputs=(cfgs.CLASS_NUM + 1) * 4,
                                                     weights_initializer=cfgs.BBOX_INITIALIZER,
                                                     activation_fn=None, trainable=self.is_training,
                                                     scope='reg_fc')

                cls_score = tf.reshape(cls_score, [-1, cfgs.CLASS_NUM + 1])
                bbox_pred = tf.reshape(bbox_pred, [-1, 4 * (cfgs.CLASS_NUM + 1)])

                return bbox_pred, cls_score

    def assign_levels(self, all_rois, labels=None, bbox_targets=None):
        '''

        :param all_rois:
        :param labels:
        :param bbox_targets:
        :return:
        '''
        with tf.name_scope('assign_levels'):
            # all_rois = tf.Print(all_rois, [tf.shape(all_rois)], summarize=10, message='ALL_ROIS_SHAPE*****')
            xmin, ymin, xmax, ymax = tf.unstack(all_rois, axis=1)

            h = tf.maximum(0., ymax - ymin)
            w = tf.maximum(0., xmax - xmin)

            levels = tf.floor(4. + tf.log(tf.sqrt(w * h + 1e-8) / 224.0) / tf.log(2.))  # 4 + log_2(***)
            # use floor instead of round

            min_level = int(cfgs.LEVLES[0][-1])
            max_level = min(5, int(cfgs.LEVLES[-1][-1]))
            levels = tf.maximum(levels, tf.ones_like(levels) * min_level)  # level minimum is 2
            levels = tf.minimum(levels, tf.ones_like(levels) * max_level)  # level maximum is 5

            levels = tf.stop_gradient(tf.reshape(levels, [-1]))

            def get_rois(levels, level_i, rois, labels, bbox_targets):

                level_i_indices = tf.reshape(tf.where(tf.equal(levels, level_i)), [-1])
                # level_i_indices = tf.Print(level_i_indices, [tf.shape(tf.where(tf.equal(levels, level_i)))[0]], message="SHAPE%d***"%level_i,
                #                            summarize=10)
                tf.summary.scalar('LEVEL/LEVEL_%d_rois_NUM' % level_i, tf.shape(level_i_indices)[0])
                level_i_rois = tf.gather(rois, level_i_indices)

                if self.is_training:
                    if cfgs.CUDA9:
                        # Note: for cuda 9
                        level_i_rois = tf.stop_gradient(level_i_rois)
                        level_i_labels = tf.gather(labels, level_i_indices)

                        level_i_targets = tf.gather(bbox_targets, level_i_indices)
                    else:

                        # Note: for cuda 8
                        level_i_rois = tf.stop_gradient(tf.concat([level_i_rois, [[0, 0, 0., 0.]]], axis=0))
                        # to avoid the num of level i rois is 0.0, which will broken the BP in tf

                        level_i_labels = tf.gather(labels, level_i_indices)
                        level_i_labels = tf.stop_gradient(tf.concat([level_i_labels, [0]], axis=0))

                        level_i_targets = tf.gather(bbox_targets, level_i_indices)
                        level_i_targets = tf.stop_gradient(tf.concat([level_i_targets,
                                                                      tf.zeros(shape=(1, 4 * (cfgs.CLASS_NUM + 1)),
                                                                               dtype=tf.float32)], axis=0))

                    return level_i_rois, level_i_labels, level_i_targets
                else:
                    if not cfgs.CUDA9:
                        # Note: for cuda 8
                        level_i_rois = tf.concat([level_i_rois, [[0, 0, 0., 0.]]], axis=0)
                    return level_i_rois, None, None

            rois_list = []
            labels_list = []
            targets_list = []
            for i in range(min_level, max_level + 1):
                P_i_rois, P_i_labels, P_i_targets = get_rois(levels, level_i=i, rois=all_rois,
                                                             labels=labels,
                                                             bbox_targets=bbox_targets)
                rois_list.append(P_i_rois)
                labels_list.append(P_i_labels)
                targets_list.append(P_i_targets)

            if self.is_training:
                all_labels = tf.concat(labels_list, axis=0)
                all_targets = tf.concat(targets_list, axis=0)
                return rois_list, all_labels, all_targets
            else:
                return rois_list  # [P2_rois, P3_rois, P4_rois, P5_rois] Note: P6 do not assign rois

    def add_anchor_img_smry(self, img, anchors, labels, method):

        positive_anchor_indices = tf.reshape(tf.where(tf.greater_equal(labels, 1)), [-1])
        negative_anchor_indices = tf.reshape(tf.where(tf.equal(labels, 0)), [-1])

        positive_anchor = tf.gather(anchors, positive_anchor_indices)
        negative_anchor = tf.gather(anchors, negative_anchor_indices)

        pos_in_img = show_box_in_tensor.only_draw_boxes(img_batch=img,
                                                        boxes=positive_anchor,
                                                        method=method)
        neg_in_img = show_box_in_tensor.only_draw_boxes(img_batch=img,
                                                        boxes=negative_anchor,
                                                        method=method)

        tf.summary.image('positive_anchor', pos_in_img)
        tf.summary.image('negative_anchors', neg_in_img)

    def add_roi_batch_img_smry(self, img, rois, labels, method):
        positive_roi_indices = tf.reshape(tf.where(tf.greater_equal(labels, 1)), [-1])

        negative_roi_indices = tf.reshape(tf.where(tf.equal(labels, 0)), [-1])

        pos_roi = tf.gather(rois, positive_roi_indices)
        neg_roi = tf.gather(rois, negative_roi_indices)

        pos_in_img = show_box_in_tensor.only_draw_boxes(img_batch=img,
                                                        boxes=pos_roi,
                                                        method=method)
        neg_in_img = show_box_in_tensor.only_draw_boxes(img_batch=img,
                                                        boxes=neg_roi,
                                                        method=method)

        tf.summary.image('pos_rois', pos_in_img)
        tf.summary.image('neg_rois', neg_in_img)

    def build_loss(self, rpn_box_pred, rpn_bbox_targets, rpn_cls_score, rpn_labels,
                   bbox_pred, bbox_targets, cls_score, labels, mask_list=None, mask_gt_list=None):

        with tf.variable_scope('build_loss'):

            if cfgs.USE_SUPERVISED_MASK:
                with tf.variable_scope("supervised_mask_loss"):
                    mask_losses = 0
                    print(mask_list)
                    print(mask_gt_list)
                    for i in range(len(mask_list)):
                        a_mask, a_mask_gt = mask_list[i], mask_gt_list[i]
                        # b, h, w, c = a_mask.shape
                        last_dim = 2 if cfgs.BINARY_MASK else cfgs.CLASS_NUM + 1
                        a_mask = tf.reshape(a_mask, shape=[-1, last_dim])
                        a_mask_gt = tf.reshape(a_mask_gt, shape=[-1])
                        a_mask_loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=a_mask,
                                                                                labels=a_mask_gt))
                        mask_losses += a_mask_loss
                    self.loss_dict['mask_loss'] = mask_losses * cfgs.SUPERVISED_MASK_LOSS_WEIGHT / float(len(mask_list))

            with tf.variable_scope('rpn_loss'):

                rpn_reg_loss = losses.smooth_l1_loss_rpn(bbox_pred=rpn_box_pred,
                                                         bbox_targets=rpn_bbox_targets,
                                                         label=rpn_labels,
                                                         sigma=cfgs.RPN_SIGMA)
                rpn_select = tf.reshape(tf.where(tf.not_equal(rpn_labels, -1)), [-1])
                rpn_cls_score = tf.reshape(tf.gather(rpn_cls_score, rpn_select), [-1, 2])
                rpn_labels = tf.reshape(tf.gather(rpn_labels, rpn_select), [-1])
                rpn_cls_loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=rpn_cls_score,
                                                                                labels=rpn_labels))

                self.loss_dict['rpn_cls_loss'] = rpn_cls_loss * cfgs.RPN_CLASSIFICATION_LOSS_WEIGHT
                self.loss_dict['rpn_reg_loss'] = rpn_reg_loss * cfgs.RPN_LOCATION_LOSS_WEIGHT

            with tf.variable_scope('FastRCNN_loss'):
                if not cfgs.FAST_RCNN_MINIBATCH_SIZE == -1:
                    reg_loss = losses.smooth_l1_loss_rcnn(bbox_pred=bbox_pred,
                                                          bbox_targets=bbox_targets,
                                                          label=labels,
                                                          num_classes=cfgs.CLASS_NUM + 1,
                                                          sigma=cfgs.FASTRCNN_SIGMA)

                    # cls_score = tf.reshape(cls_score, [-1, cfgs.CLASS_NUM + 1])
                    # labels = tf.reshape(labels, [-1])
                    cls_loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(
                        logits=cls_score,
                        labels=labels))  # beacause already sample before
                else:
                    '''
                    applying OHEM here
                    '''
                    print(20 * "@@")
                    print("@@" + 10 * " " + "TRAIN WITH OHEM ...")
                    print(20 * "@@")
                    cls_loss, reg_loss = losses.sum_ohem_loss(cls_score=cls_score,
                                                              label=labels,
                                                              bbox_targets=bbox_targets,
                                                              bbox_pred=bbox_pred,
                                                              num_ohem_samples=256,
                                                              num_classes=cfgs.CLASS_NUM + 1,
                                                              sigma=cfgs.FASTRCNN_SIGMA)

                self.loss_dict['fast_cls_loss'] = cls_loss * cfgs.FAST_RCNN_CLASSIFICATION_LOSS_WEIGHT
                self.loss_dict['fast_reg_loss'] = reg_loss * cfgs.FAST_RCNN_LOCATION_LOSS_WEIGHT

    def build_whole_detection_network(self, input_img_batch, gtboxes_batch, gtboxes_r_batch):

        if self.is_training:
            # ensure shape is [M, 5]
            gtboxes_batch = tf.reshape(gtboxes_batch, [-1, 5])
            gtboxes_batch = tf.cast(gtboxes_batch, tf.float32)

            gtboxes_r_batch = tf.reshape(gtboxes_r_batch, [-1, 6])
            gtboxes_r_batch = tf.cast(gtboxes_r_batch, tf.float32)

        img_shape = tf.shape(input_img_batch)

        # 1. build base network
        mask_list = []
        if cfgs.USE_SUPERVISED_MASK:
            P_list, mask_list = self.build_base_network(input_img_batch)  # [P2, P3, P4, P5, P6], [mask_p2, mask_p3]
        else:
            P_list = self.build_base_network(input_img_batch)  # [P2, P3, P4, P5, P6]

        # 2. build rpn
        with tf.variable_scope('build_rpn',
                               regularizer=slim.l2_regularizer(cfgs.WEIGHT_DECAY)):

            fpn_cls_score = []
            fpn_box_pred = []
            for level_name, p in zip(cfgs.LEVLES, P_list):
                if cfgs.SHARE_HEADS:
                    reuse_flag = None if level_name == cfgs.LEVLES[0] else True
                    scope_list = ['rpn_conv/3x3', 'rpn_cls_score', 'rpn_bbox_pred']
                else:
                    reuse_flag = None
                    scope_list = ['rpn_conv/3x3_%s' % level_name, 'rpn_cls_score_%s' % level_name,
                                  'rpn_bbox_pred_%s' % level_name]
                rpn_conv3x3 = slim.conv2d(
                    p, 512, [3, 3],
                    trainable=self.is_training, weights_initializer=cfgs.INITIALIZER, padding="SAME",
                    activation_fn=tf.nn.relu,
                    scope=scope_list[0],
                    reuse=reuse_flag)
                rpn_cls_score = slim.conv2d(rpn_conv3x3, self.num_anchors_per_location * 2, [1, 1], stride=1,
                                            trainable=self.is_training, weights_initializer=cfgs.INITIALIZER,
                                            activation_fn=None, padding="VALID",
                                            scope=scope_list[1],
                                            reuse=reuse_flag)
                rpn_box_pred = slim.conv2d(rpn_conv3x3, self.num_anchors_per_location * 4, [1, 1], stride=1,
                                           trainable=self.is_training, weights_initializer=cfgs.BBOX_INITIALIZER,
                                           activation_fn=None, padding="VALID",
                                           scope=scope_list[2],
                                           reuse=reuse_flag)
                rpn_box_pred = tf.reshape(rpn_box_pred, [-1, 4])
                rpn_cls_score = tf.reshape(rpn_cls_score, [-1, 2])

                fpn_cls_score.append(rpn_cls_score)
                fpn_box_pred.append(rpn_box_pred)

            fpn_cls_score = tf.concat(fpn_cls_score, axis=0, name='fpn_cls_score')
            fpn_box_pred = tf.concat(fpn_box_pred, axis=0, name='fpn_box_pred')
            fpn_cls_prob = slim.softmax(fpn_cls_score, scope='fpn_cls_prob')

        # 3. generate_anchors
        all_anchors = []
        mask_gt_list = []
        for i in range(len(cfgs.LEVLES)):
            level_name, p = cfgs.LEVLES[i], P_list[i]

            p_h, p_w = tf.shape(p)[1], tf.shape(p)[2]
            if cfgs.USE_SUPERVISED_MASK and i < len(mask_list) and self.is_training:
                if cfgs.MASK_TYPE.strip() == 'h':
                    mask = tf.py_func(mask_utils.make_gt_mask,
                                      [p_h, p_w, img_shape[1], img_shape[2], gtboxes_batch],
                                      Tout=tf.int32)
                elif cfgs.MASK_TYPE.strip() == 'r':
                    mask = tf.py_func(mask_utils.make_r_gt_mask,
                                      [p_h, p_w, img_shape[1], img_shape[2], gtboxes_r_batch],
                                      Tout=tf.int32)
                if cfgs.BINARY_MASK:
                    mask = tf.where(tf.greater(mask, 0), tf.ones_like(mask), tf.zeros_like(mask))
                mask_gt_list.append(mask)
                mask_utils.vis_mask_tfsmry(mask, name="MASK/%s" % level_name)

            featuremap_height = tf.cast(p_h, tf.float32)
            featuremap_width = tf.cast(p_w, tf.float32)
            anchors = anchor_utils.make_anchors(base_anchor_size=cfgs.BASE_ANCHOR_SIZE_LIST[i],
                                                anchor_scales=cfgs.ANCHOR_SCALES,
                                                anchor_ratios=cfgs.ANCHOR_RATIOS,
                                                featuremap_height=featuremap_height,
                                                featuremap_width=featuremap_width,
                                                stride=cfgs.ANCHOR_STRIDE_LIST[i],
                                                name="make_anchors_for%s" % level_name)
            all_anchors.append(anchors)
        all_anchors = tf.concat(all_anchors, axis=0, name='all_anchors_of_FPN')

        # 4. postprocess rpn proposals. such as: decode, clip, NMS
        with tf.variable_scope('postprocess_FPN'):
            rois, roi_scores = postprocess_rpn_proposals(rpn_bbox_pred=fpn_box_pred,
                                                         rpn_cls_prob=fpn_cls_prob,
                                                         img_shape=img_shape,
                                                         anchors=all_anchors,
                                                         is_training=self.is_training)

        if self.is_training:
            with tf.variable_scope('sample_anchors_minibatch'):
                fpn_labels, fpn_bbox_targets = \
                    tf.py_func(
                        anchor_target_layer,
                        [gtboxes_batch, img_shape, all_anchors],
                        [tf.float32, tf.float32])
                fpn_bbox_targets = tf.reshape(fpn_bbox_targets, [-1, 4])
                fpn_labels = tf.to_int32(fpn_labels, name="to_int32")
                fpn_labels = tf.reshape(fpn_labels, [-1])
                self.add_anchor_img_smry(input_img_batch, all_anchors, fpn_labels, method=0)

            # --------------------------------------add smry-----------------------------------------------------------

            fpn_cls_category = tf.argmax(fpn_cls_prob, axis=1)
            kept_rpppn = tf.reshape(tf.where(tf.not_equal(fpn_labels, -1)), [-1])
            fpn_cls_category = tf.gather(fpn_cls_category, kept_rpppn)
            acc = tf.reduce_mean(tf.to_float(tf.equal(fpn_cls_category,
                                                      tf.to_int64(tf.gather(fpn_labels, kept_rpppn)))))
            tf.summary.scalar('ACC/fpn_accuracy', acc)

            with tf.control_dependencies([fpn_labels]):
                with tf.variable_scope('sample_RCNN_minibatch'):
                    rois, labels, bbox_targets = \
                        tf.py_func(proposal_target_layer,
                                   [rois, gtboxes_batch],
                                   [tf.float32, tf.float32, tf.float32])
                    rois = tf.reshape(rois, [-1, 4])
                    labels = tf.to_int32(labels)
                    labels = tf.reshape(labels, [-1])
                    bbox_targets = tf.reshape(bbox_targets, [-1, 4 * (cfgs.CLASS_NUM + 1)])
                    self.add_roi_batch_img_smry(input_img_batch, rois, labels, method=0)

        if not cfgs.USE_CONCAT:
            if self.is_training:
                rois_list, labels, bbox_targets = self.assign_levels(all_rois=rois,
                                                                     labels=labels,
                                                                     bbox_targets=bbox_targets)
            else:
                rois_list = self.assign_levels(all_rois=rois)  # rois_list: [P2_rois, P3_rois, P4_rois, P5_rois]

        # -------------------------------------------------------------------------------------------------------------#
        #                                            Fast-RCNN                                                         #
        # -------------------------------------------------------------------------------------------------------------#

        # 5. build Fast-RCNN
        if not cfgs.USE_CONCAT:
            bbox_pred, cls_score = self.build_fastrcnn(P_list=P_list, rois_list=rois_list,
                                                       img_shape=img_shape)
            rois = tf.concat(rois_list, axis=0, name='concat_rois')
        else:
            bbox_pred, cls_score = self.build_concat_fastrcnn(P_list=P_list, all_rois=rois, img_shape=img_shape)

        cls_prob = slim.softmax(cls_score, 'cls_prob')

        # ----------------------------------------------add smry-------------------------------------------------------
        if self.is_training:
            cls_category = tf.argmax(cls_prob, axis=1)
            fast_acc = tf.reduce_mean(tf.to_float(tf.equal(cls_category, tf.to_int64(labels))))
            tf.summary.scalar('ACC/fast_acc', fast_acc)

        #  6. postprocess_fastrcnn
        if self.is_training:
            self.build_loss(rpn_box_pred=fpn_box_pred,
                            rpn_bbox_targets=fpn_bbox_targets,
                            rpn_cls_score=fpn_cls_score,
                            rpn_labels=fpn_labels,
                            bbox_pred=bbox_pred,
                            bbox_targets=bbox_targets,
                            cls_score=cls_score,
                            labels=labels,
                            mask_list=mask_list if cfgs.USE_SUPERVISED_MASK else None,
                            mask_gt_list=mask_gt_list if cfgs.USE_SUPERVISED_MASK else None)

        final_bbox, final_scores, final_category = self.postprocess_fastrcnn(rois=rois,
                                                                             bbox_ppred=bbox_pred,
                                                                             scores=cls_prob,
                                                                             img_shape=img_shape)
        if self.is_training:
            return final_bbox, final_scores, final_category, self.loss_dict
        else:
            return final_bbox, final_scores, final_category

    def get_restorer(self):
        checkpoint_path = tf.train.latest_checkpoint(os.path.join(cfgs.TRAINED_CKPT, cfgs.VERSION))

        if checkpoint_path != None:
            restorer = tf.train.Saver()
            print("model restore from :", checkpoint_path)
        else:
            checkpoint_path = cfgs.PRETRAINED_CKPT
            print("model restore from pretrained mode, path is :", checkpoint_path)

            model_variables = slim.get_model_variables()

            # for var in model_variables:
            #     print(var.name)
            # print(20*"__++__++__")

            def name_in_ckpt_rpn(var):
                return var.op.name

            def name_in_ckpt_fastrcnn_head(var):
                '''
                Fast-RCNN/resnet_v1_50/block4 -->resnet_v1_50/block4
                Fast-RCNN/MobilenetV2/** -- > MobilenetV2 **
                :param var:
                :return:
                '''
                return '/'.join(var.op.name.split('/')[1:])

            nameInCkpt_Var_dict = {}
            for var in model_variables:
                if var.name.startswith(self.base_network_name):
                    var_name_in_ckpt = name_in_ckpt_rpn(var)
                    nameInCkpt_Var_dict[var_name_in_ckpt] = var
            restore_variables = nameInCkpt_Var_dict
            for key, item in restore_variables.items():
                print("var_in_graph: ", item.name)
                print("var_in_ckpt: ", key)
                print(20 * "___")
            restorer = tf.train.Saver(restore_variables)
            print(20 * "****")
            print("restore from pretrained_weighs in IMAGE_NET")
        return restorer, checkpoint_path

    def get_gradients(self, optimizer, loss):
        '''

        :param optimizer:
        :param loss:
        :return:

        return vars and grads that not be fixed
        '''

        # if cfgs.FIXED_BLOCKS > 0:
        #     trainable_vars = tf.trainable_variables()
        #     # trained_vars = slim.get_trainable_variables()
        #     start_names = [cfgs.NET_NAME + '/block%d'%i for i in range(1, cfgs.FIXED_BLOCKS+1)] + \
        #                   [cfgs.NET_NAME + '/conv1']
        #     start_names = tuple(start_names)
        #     trained_var_list = []
        #     for var in trainable_vars:
        #         if not var.name.startswith(start_names):
        #             trained_var_list.append(var)
        #     # slim.learning.train()
        #     grads = optimizer.compute_gradients(loss, var_list=trained_var_list)
        #     return grads
        # else:
        #     return optimizer.compute_gradients(loss)
        return optimizer.compute_gradients(loss)

    def enlarge_gradients_for_bias(self, gradients):

        final_gradients = []
        with tf.variable_scope("Gradient_Mult") as scope:
            for grad, var in gradients:
                scale = 1.0
                if cfgs.MUTILPY_BIAS_GRADIENT and './biases' in var.name:
                    scale = scale * cfgs.MUTILPY_BIAS_GRADIENT
                if not np.allclose(scale, 1.0):
                    grad = tf.multiply(grad, scale)
                final_gradients.append((grad, var))
        return final_gradients


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split
import lightgbm as lgb
from sklearn.preprocessing import LabelEncoder

flight_traffic = pd.read_csv("C:\\Users\\James\\Documents\\ML\\Citadel Summer Invitation 2018 NYC\\flight_traffic.csv", na_values=[0])
stocks = pd.read_csv("C:\\Users\\James\\Documents\\ML\\Citadel Summer Invitation 2018 NYC\\stock_prices.csv", na_values=[0])

stocks['Date'] = pd.to_datetime(stocks['timestamp'])
flight_traffic["total_delay"] = flight_traffic["airline_delay"] + flight_traffic["weather_delay"] + flight_traffic["air_system_delay"] + flight_traffic["security_delay"] + flight_traffic["aircraft_delay"]
flight_traffic = flight_traffic.assign(Date=pd.to_datetime(flight_traffic[['year', 'month', 'day']]))
flight_traffic.set_index(pd.to_datetime(flight_traffic['Date']), inplace=True)
merged = flight_traffic.merge(stocks,on='Date',how='left')

filtered = {}
for airline in merged.airline_id.unique():
    if airline in merged.columns:
        filtered[airline] = merged[merged["airline_id"] == airline]
        filtered[airline] = filtered[airline].interpolate()
        filtered[airline]["stock_price"] =  filtered[airline][airline]

        le = LabelEncoder()
        for feature in ['origin_airport',  'destination_airport']:
            filtered[airline][feature] = le.fit_transform(filtered[airline][feature])
            y = filtered[airline]['stock_price']
        #y['stock_price'] = (y['stock_price'] - y['stock_price'].mean()) / y['stock_price'].std(); #print(X['total_delay'])

mapes = {}
for airline in ['AA',	'UA',	'B6',	'OO',	'AS',	'NK',	'WN',	'DL',	'HA']:
    target = airline
    X = filtered[airline].drop(["stock_price"],axis=1)[['year', 'month', 'origin_airport', 'destination_airport', 'cancelled', 'diverted', 'total_delay']]
    X['total_delay'] = (X['total_delay'] - X['total_delay'].mean()) / X['total_delay'].std(); #print(X['total_delay'])
    y = filtered[airline]['stock_price']

    X_train, X_test, y_train, y_test = train_test_split(X,y)
    X_train, X_valid, y_train, y_valid = train_test_split(X_train,y_train)
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_test = lgb.Dataset(X_test, y_test)
    lgb_valid = lgb.Dataset(X_valid, y_valid)

    bst = lgb.train(core_params, lgb_train, num_round, valid_sets=[lgb_valid])
    ypred = bst.predict(X_test, num_iteration=bst.best_iteration)
    mapes[airline] = mean_absolute_percentage_error(y_test,ypred)

core_params = {
    'boosting_type': 'gbdt', # GBM type: gradient boosted decision tree, rf (random forest), dart, goss.
    'objective': 'regression', # the optimization object: binary, regression, multiclass, xentropy.
    'learning_rate': 0.01, # the gradient descent learning or shrinkage rate, controls the step size.
    'num_leaves': 5, # the number of leaves in one tree.
    'nthread': 4, # number of threads to use for LightGBM, best set to number of actual cores.

    'metric': 'mape' # an additional metric to calculate during validation: area under curve (auc).
}

num_round = 1000

def mean_absolute_percentage_error(y_true, y_pred):
    return np.mean(np.abs((y_true - y_pred) / y_true)) * 1

print(mean_absolute_percentage_error(y_test,ypred))
import graphviz
bst.save_model('model.txt')
lgb.plot_tree(bst, figsize=(20, 20))


import os

import ls

exitWords = ["exit", "quit"]
while True:
    path = os.path.abspath('.')
    print(path, "$ ", end='')
    inputedMessage = input()
    resultList = inputedMessage.rstrip().lstrip().split(" ")
    resultDirection = resultList[0]
    if resultDirection == '':
        pass
    elif resultDirection == "ls":
        ls.ls(path)
    elif resultDirection in exitWords:
        break
    elif resultDirection == "dir":
        for i in os.listdir(os.path.abspath('.')):
            print(i)
    else:
        print("Unknown direction:", resultDirection)


#! /usr/bin/env python3

class Character():
    # This is the main character class, not only for player characters
    def __init__(self, name, race, kon, srg, ges, agi):
        self.name = name
        self.race = race
        self.kon = kon  # Constitution
        self.srg = srg  # Strength
        self.ges = ges  # ABility
        self.agi = agi  # AGility
        self.hp = self.kon * 8 + self.srg * 5 + 10  # Health Points

    level = 1
    experience = 0

    def print_char(self, name, race, kon, srg, ges, agi, hp, level):
        # print out character
        print("----- Spielercharakter -----")
        print("NAME: {}   LEVEL: {}   RASSE: {}".format(name, str(level), race))
        print('WERTE: ')
        print("Leben: {} / {}".format(str(hp), str(kon * 8 + srg * 5 + 10)))
        print("Bewegung: {}".format(str(movement(agi, kon, race))))
        print("Nahkampf: {}".format(str(melee_skill(srg, ges))))
        print("Fernkampf: {}".format(str(gun_skill(agi, ges))))
        print("ATTRIBUTE: ".format())
        print("Konstitution: {}".format(str(kon)))
        print("Stärke: {}".format(str(srg)))
        print("Geschicklichkeit: {}".format(str(ges)))
        print("Schnelligkeit: {}".format(str(agi)))
        print("----------------------------")

'''	def print_char(self, name, race, kon, srg, ges, agi):
		print("----- Spielercharakter -----")
		print("NAME: " + self.name + "   LEVEL: " + str(self.level))
		print("WERTE:")
		print("Leben: " + str(self.hp) + " / " + str(self.kon*8 + self.srg*5 + 10))
		print("Bewegung: " + str(movement(self.agi, self.kon, self.race)))
		print("Nahkampf: " + str(melee_skill(self.srg, self.ges)))
		print("Fernkampf: " + str(gun_skill(self.agi, self.ges)))
		print("ATTRIBUTE: ")
		print("Konstitution: " + str(self.kon))
		print("Stärke: " + str(self.srg))
		print("Geschicklichkeit: " + str(self.ges))
		print("Schnelligkeit: " + str(self.agi))
		print("----------------------------") '''

class Vampire(Character):
    # race subclass
    def __init__(self, name):
        super().__init__(name, "Vampir", 6, 8, 5, 5)

class Angel(Character):
    # race subclass
    def __init__(self, name):
        super().__init__(name, "Engel", 4, 3, 7, 7)

class Human(Character):
    # race subclass
    def __init__(self, name):
        super().__init__(name, "Mensch", 4, 5, 5, 4)

class Werewolf(Character):
    # race subclass
    def __init__(self, name):
        super().__init__(name, "Werwolf", 4, 5, 5, 4)
        self.morph = True

def race_choice(new_race):
    # race choice function
    if new_race == "1":
        new_race = "Vampir"
    elif new_race == "2":
        new_race = "Werwolf"
    elif new_race == "3":
        new_race = "Engel"
    else:
        # this is for wrong input, but i think this is a bad solution
        print('Erz\xfcrne IHN nicht mit einer falschen Angabe!')
        new_race = input("Welcher Rasse soll es angehoeren? 1: Vampir \n 2: Werwolf \n 3:Engel")
        race_choice(new_race)
    return new_race

def char_create():
    # Player Character Creation. here is the place where legends are born. and taunted.
    print("Und wieder schickt ER einen seiner Diener auf die Erde.")
    print("Wie es schon tausendmal passiert ist, ohne dass der gewoehnliche Mensch davon etwas mitbekommen würde.")
    print("Wie alle anderen Astralwesen davor hat er nur ein Ziel...")
    input("Press Enter to continue...")
    print()
    print("UNHEIL.")
    print()
    input("Press enter to continue")
    new_name = input("Wie soll das neue Wesen heissen? ")
    new_race = input("Welcher Rasse soll es angehoeren? \n 1: Vampir \n 2: Werwolf \n 3: Engel \n")
    new_race = race_choice(new_race)
    print()
    if new_race == "Vampir":
        print("Eine weise Wahl. Du Fetischist.")
        player = Vampire(new_name)
    elif new_race == "Werwolf":
        print("Aah, ein Rowdy!")
        player = Werewolf(new_name)
    else:
        print("Schwuchtel.")
        player = Angel(new_name)
    print()
    print()
    print("Du erwachst nach einem langen Flug...")
    print()
    return player

def movement(agi, kon, race):
    if race == "Engel":
        return int((kon * 3 + agi * 8) / 10) + 1
    else:
        return int((kon * 3 + agi * 8) / 10)

def melee_skill(srg, ges):
    return int(srg * 5 + ges * 3)

def gun_skill(agi, ges):
    return int(agi * 3 + ges * 7)

def level_up(level, experience):
    # has to be called every time the player earns experience. need to add a skill system in this function
    if experience >= level * 5 + 20:
        experience -= level * 5 + 20
        level += 1
        lst = [level, experience]
        return lst

def main_menu():
    # why is this still in english?
    print("   ####################")
    print("#### Treasure Hunting ####")
    print("   ####################")
    print()
    print("(n)ew game")
    print("(l)oad game")
    print("(e)xit")
    i = input()
    if i == "n":
        start_game()
    if i == "l":
        pass
    # load savegame file and proceed to the actual game?
    if i == "e":
        print("Good bye!")
        quit()
    else:
        # goddamn, i really need a better solution for this...
        main_menu()

# Turn
def your_turn(player):
    move = True
    shoot_fight = True
    if player.morph == True:
        morphed = input("Willst du dich verwandeln? y/n")
        if morphed == "y":
            morphed = True
            move = False
            shoot_fight = False
        else:
            morphed = False
    if move == True:
        print("Wohin willst du gehen? du hast %s Schritte." % (str(movement(player))))
        s = [input("wasd: ")]
        steps = []
        for i in range(movement(player)):
            if s[i] in "wasd":
                steps.append(s[i])
            # call walk-function with steps-list
    if shoot_fight == True:
        foes = show_foes()
        print("Wen willst du angreifen?")
        print("0: nichts tun")
        for i in foes:
            if near(foes[i]) == True:
                # is the enemy in range? shoot or fight him? maybe you cant shoot if you are in direct combat?
                print(str(foes[i]) + ": angreifen")
            elif sight(foes[i]) == True:
                # is the enemy in sight line? then you can shoot him:
                print(str(foes[i]) + ": schiessen")
        action = input()
        attack(action)  # function has to be build
    print("Deine Gegner sind am Zug...")
    print()

def show_foes():
    # returns a list of enemies on the map
    pass

'''
now we could use a simple gui to display the battlefield in a different window - lets take tkinter, even if i have no clue how to use it <3

# print out the battlefield - GUI?
from tkinter import *
def new_window():
	# das Fenster selbst festlegen
	top = Tk()
	F = Frame(top)
	F.pack()
	F.master.title("Schlachtfeld - Treasure Hunting")

	# die Widgets hinzufügen
	lHello = Label(F, text="Hier sollte man eig das Schlachtfeld sehen...")
	lHello.pack()
	bQuit = Button(F, text="Quit", command=F.quit)
	bQuit.pack()

	# bringt die Schleife ans Laufen
	top.mainloop()
'''

'''
OK, enough with functions and classes!
let's get started with the actual game!
'''

def start_game():  # is called by main_menu-function - base architecture?
    player = char_create()
    player.print_char(player.name, player.race, player.kon, player.srg, player.ges, player.agi, player.hp, player.level)
    main_menu()

main_menu()


class Evaluation:

    total_true_positives = 0
    total_false_positives = 0
    total_false_negatives = 0

    macro_precision = 0
    macro_recall = 0
    macro_f1 = 0

    micro_precision = 0
    micro_recall = 0
    micro_f1 = 0

    n_samples = 0

    def __init__(self, method):
        self.total_true_positives = 0
        self.total_false_positives = 0
        self.total_false_negatives = 0

        self.macro_precision = 0
        self.macro_recall = 0
        self.macro_f1 = 0

        self.micro_precision = 0
        self.micro_recall = 0
        self.micro_f1 = 0

        self.default_method=method

    def get_average_positives(self):
        return (self.total_false_positives + self.total_true_positives)/self.n_samples

    def summarize(self, method=None):
        if method is None:
            method = self.default_method

        string = ""
        if method == "micro":
            string += "Final results (micro-averaged):\n"
            string += "===============================\n"
            string += "Precision: " + str(self.micro_precision) + "\n"
            string += "Recall: " + str(self.micro_recall) + "\n"
            string += "F1: " + str(self.micro_f1) + "\n"
            string += "Average positives: " + str(self.get_average_positives()) + "\n"
        else:
            string += "Final results (macro-averaged):\n"
            string += "===============================\n"
            string += "Precision: " + str(self.macro_precision) + "\n"
            string += "Recall: " + str(self.macro_recall) + "\n"
            string += "F1: " + str(self.macro_f1) + "\n"
            string += "Average positives: " + str(self.get_average_positives()) + "\n"
        return string


from sys import argv
from random import randint
import datetime
#import datetime
#script, name, game, tea, date = argv
#
#date = datetime()
#
#print("The name of this file is:", script)
#print("Jordan is my", name)
#print("Zelda is the", game, "I am playing.")
#print("I am drinking", tea)
#print("The date is:", date)

number = input("Please enter a number: ")
random_number = randint(1, 101)
d = datetime.datetime.now()
hello = argv

print("I am going to ask for a number:", random_number, d)

number = input("Please enter a number")
random_number =  randint(1, 101)
d = datetime.datetime.now()
hello = argv


import json
import re
from collections import Counter

pattern_line = re.compile(r"^\{'points': \'?(?P<points>[^\'\,]+)\'?, "
                          r"'title': \'?[^\'\,]+\'?, "
                          r"'description': \"[^\"]+\", "
                          r"'taster_name': \'?(?P<taster_name>[^\'\,]+)\'?, "
                          r"'taster_twitter_handle': \'?[^\'\,]+\'?, "
                          r"'price': \'?(?P<price>[^\'\,]+)\'?, "
                          r"'designation': \'?[^\'\,]+\'?, "
                          r"'variety': \'?(?P<variety>[^\'\,]+)\'?, "
                          r"'region_1': \'?(?P<region_1>[^\'\,]+)\'?, "
                          r"'region_2': \'?(?P<region_2>[^\'\,]+)\'?, "
                          r"'province': \'?[^\'\,]+\'?, "
                          r"'country': \'?(?P<country>[^\'\,]+)\'?, "
                          r"'winery': \'?[^\,\']+\'?\}$")

def get_whole_data(data_1, data_2):
    with open(data_1) as d1, open(data_2) as d2:
        loading_data = json.load(d1)
        loading_data.extend(json.load(d2))

    unique_items_set = set()
    for dict_item in loading_data:
        unique_items_set.add(tuple(dict_item.items()))

    dumping_data = [dict(item) for item in unique_items_set]
    dumping_data.sort(key=lambda item: (0-(item['price'] or 0), item['variety'] or 'z'))

    full_winedata = 'winedata_full.json'
    with open(full_winedata, 'w') as d:
        json.dump(dumping_data, d, ensure_ascii=False, indent='\t')

    return full_winedata

def create_report_wine_dict(data):
    wine_dict = {}

    for item in data:
        if item["variety"] and item["variety"] != "None" and item["variety"] != 'None':
            if item["variety"] not in wine_dict:
                wine_dict[item["variety"]] = {'points': [],
                                              'price': [],
                                              'region': [],
                                              'country': []}
            if item["points"] is not None:
                wine_dict[item["variety"]]['points'].append(item["points"])
            if item["price"] is not None:
                wine_dict[item["variety"]]['price'].append(item["price"])
            if item["region_1"] is not None and item["region_1"] != "None" and item["region_1"] != 'None':
                wine_dict[item["variety"]]['region'].append(item["region_1"])
            if item["region_2"] is not None and item["region_2"] != "None" and item["region_2"] != 'None':
                wine_dict[item["variety"]]['region'].append(item["region_2"])
            if item["country"] is not None and item["country"] != "None" and item["country"] != 'None':
                wine_dict[item["variety"]]['country'].append(item["country"])

    return wine_dict

def create_report_country_dict(data):
    country_dict = {}

    for item in data:
        if item["country"] and item["country"] != "None" and item["country"] != 'None':
            if item["country"] not in country_dict:
                country_dict[item["country"]] = {'points': [],
                                                 'price': []}

            if item["points"] is not None:
                country_dict[item["country"]]['points'].append(item["points"])
            if item["price"] is not None:
                country_dict[item["country"]]['price'].append(item["price"])

    return country_dict

def get_wine_stat(data):
    dict_data = create_report_wine_dict(data)
    wine_stat_dict = {}
    wines = ['Gewürztraminer', 'Riesling', 'Merlot', 'Madera', 'Tempranillo', 'Red Blend']

    for wine in wines:
        if wine in dict_data:
            wine_stat_dict[wine] = \
                {'avarege_price': round((sum(dict_data[wine]['price'])/len(dict_data[wine]['price']) or 0), 2),
                 'min_price': sorted(dict_data[wine]['price'])[0],
                 'max_price': sorted(dict_data[wine]['price'])[-1],
                 'most_common_region': Counter(dict_data[wine]['region']).most_common(1)[0][0],
                 'most_common_country': Counter(dict_data[wine]['country']).most_common(1)[0][0],
                 'avarage_score': round((sum(dict_data[wine]['points'])/len(dict_data[wine]['points'])), 2)
                 }

    return wine_stat_dict

def get_country_stat(data):
    dict_data = create_report_country_dict(data)
    for item in dict_data:
        dict_data[item]['avg_price'] = (sum(dict_data[item]['price'])/len(dict_data[item]['price'])
                                        if len(dict_data[item]['price']) > 0 else 0)
        dict_data[item]['avg_score'] = sum(dict_data[item]['points']) / len(dict_data[item]['points'])

    avg_price_list = []
    avg_score_list = []

    for item in dict_data:
        avg_price_list.append([dict_data[item]['avg_price'], item])
        avg_score_list.append([dict_data[item]['avg_score'], item])

    avg_price_list.sort()
    avg_score_list.sort()

    most_expensive_country = [i[1] for i in avg_price_list if i[0] == avg_price_list[-1][0]]
    cheapest_country = [i[1] for i in avg_price_list if i[0] == avg_price_list[0][0]]
    most_rated_country = [i[1] for i in avg_score_list if i[0] == avg_score_list[-1][0]]
    underrated_country = [i[1] for i in avg_score_list if i[0] == avg_score_list[0][0]]

    country_stat_dict = \
        {'most_expensive_country': most_expensive_country,
         'cheapest_country': cheapest_country,
         'most_rated_country': most_rated_country,
         'underrated_country': underrated_country
         }

    return country_stat_dict

def parse_item(record):
    match_line = pattern_line.match(record)
    if match_line:
        try:
            points = int(match_line.groupdict()['points'].strip(''""))
        except ValueError:
            points = None
        taster_name = match_line.groupdict()['taster_name'].strip(''"")
        try:
            price = float(match_line.groupdict()['price'].strip(''""))
        except ValueError:
            price = None
        variety = match_line.groupdict()['variety'].strip(''"")
        region_1 = match_line.groupdict()['region_1'].strip(''"")
        region_2 = match_line.groupdict()['region_2'].strip(''"")
        country = match_line.groupdict()['country'].strip(''"")
        return points, taster_name, price, variety, region_1, region_2, country
    return

def read_data_file(file_name):
    with open(file_name) as f:
        for item in json.load(f):
            record_item = parse_item(str(item))
            if record_item:
                yield record_item

def get_statistics(data_file):
    stat_data = []
    records = read_data_file(data_file)
    for i in records:
        stat_data.append({"points": i[0], "taster_name": i[1], "price": i[2], "variety": i[3],
                          "region_1": i[4], "region_2": i[5], "country": i[6]})

    stat_data.sort(key=lambda item: item['price'] or 0, reverse=True)

    most_expensive_wine_set = set([x['variety'] for x in stat_data if x['price'] == stat_data[0]['price']])
    most_expensive_wine = [item for item in most_expensive_wine_set]

    data_without_null_price = list(filter(lambda item: item['price'], stat_data))

    cheapest_wine_set = set([x['variety'] for x in data_without_null_price
                             if x['price'] == data_without_null_price[-1]['price']])
    cheapest_wine = [item for item in cheapest_wine_set]

    stat_data.sort(key=lambda item: item['points'] or 0)

    highest_score_set = set([x['variety'] for x in stat_data if x['points'] == stat_data[-1]['points']])
    highest_score = [item for item in highest_score_set]

    data_without_null_score = list(filter(lambda item: item['points'], stat_data))

    lowest_score_set = set([x['variety'] for x in data_without_null_score if x['points'] == stat_data[0]['points']])
    lowest_score = [item for item in lowest_score_set]

    most_active_commentator = Counter(
        item['taster_name'] for item in stat_data
        if item['taster_name'] and item['taster_name'] != "None" and item['taster_name'] != 'None').most_common(1)[0][0]

    wine_stat_dict = get_wine_stat(stat_data)
    country_stat_dict = get_country_stat(stat_data)

    statictics = {"statistics": {
                    "wine": wine_stat_dict,
                    "most_expensive_wine": most_expensive_wine,
                    "cheapest_wine": cheapest_wine,
                    "highest_score": highest_score,
                    "lowest_score": lowest_score,
                    "most_expensive_country": country_stat_dict["most_expensive_country"],
                    "cheapest_country": country_stat_dict["cheapest_country"],
                    "most_rated_country": country_stat_dict["most_rated_country"],
                    "underrated_country": country_stat_dict["underrated_country"],
                    "most_active_commentator": most_active_commentator
                  }
    }

    return statictics

def render_report(data):
    with open('stats.json', 'w+') as sj:
        json.dump(data, sj, ensure_ascii=False, indent='\t')

    with open('stats.MD', 'w+') as md:
        md.write('## wine statistics: \n')
        md.write("```\n")
        json.dump(data, md, ensure_ascii=False, indent='\t')
        md.write('\n')
        md.write("```")

def main(winedata_1, winedata_2):
    wine_data = get_whole_data(winedata_1, winedata_2)
    wine_report = get_statistics(wine_data)
    render_report(wine_report)

if __name__ == '__main__':
    datafile_1 = 'winedata_1.json'
    datafile_2 = 'winedata_2.json'
    main(datafile_1, datafile_2)


import datetime
import json
import os
import re
from urllib.request import urlopen
from bs4 import BeautifulSoup

#problem: how to solve overlapping issues?
#1. can use local dict and keep track, but will double the memory size (3000ish max)
#2. somehow use json properties? but how to do this?
#3. use dict to store all data, then convert it to json later?

cb_version_url = "http://bemaniwiki.com/index.php?beatmania%20IIDX%2025%20CANNON%20BALLERS"
main_page = urlopen(cb_version_url)
version_name = BeautifulSoup(main_page, "html5lib").findAll('strong', text=re.compile("^LDJ:J:B:A:"))[0].text[-10:]
print(version_name)

cb_new_url = "http://bemaniwiki.com/index.php?beatmania%20IIDX%2025%20CANNON%20BALLERS%2F%BF%B7%B6%CA%A5%EA%A5%B9%A5%C8"
cb_old_url = "http://bemaniwiki.com/index.php?beatmania%20IIDX%2025%20CANNON%20BALLERS%2F%B5%EC%B6%CA%A5%EA%A5%B9%A5%C8"

page_new = urlopen(cb_new_url)
page_old = urlopen(cb_old_url)

print ("Opened pages")

song_new_table = BeautifulSoup(page_new, "html5lib").find('div', class_='ie5')
song_old_table = BeautifulSoup(page_old, "html5lib").find_all('div', class_='ie5')[1]

cb_new_rows = song_new_table.find_all('tr')
cb_old_rows = song_old_table.find_all('tr')

print ("Parsed pages")

songs = []

song_dict = {}

#solution: use dictionary, have title + difficulty (in plain text, i.e. SPA) as the key
#REMINDER: leggendarias are separate difficulty due to reasons
#also, remove (HCN ver.) before creating key

#reminder to self for DDR: BSP, two of them exist. differentiate beginner

print ("Grabbing data...")

def get_level(col):
    level = col.text
    if len(col) != 1 and level != '-' and level != '' and len(col) != 0:
        index = len(level)-1
        end_index = index
        while (level[index] != ']'):
            index = index-1
        level = level[index+1:len(level)]
    elif level == '-' or level == '':
        level = -1
    return level

diff_options = {
    0: "basic",
    1: "novice",
    2: "hyper",
    3: "another",
    4: "leggendaria"
}

def get_song(level, difficulty, version, style, title, artist, genre, bpm):
    if(level != -1):
        diff_name = diff_options[difficulty]
        key = title + " " + artist + " " + version + " " + bpm + " "+ diff_name + " " + style
        if key not in song_dict:
            data = {
                "title": title,
                "artist": artist,
                "genre": genre,
                "bpm": bpm,
                "style": style,
                "difficulty": difficulty,
                "level": level,
                "version": version
            }
            song_dict[key] = True
            songs.append(data)

leggendaria = "LEGGENDARIA"
leggendaria_mark = "†"
hcn = "(HCN Ver.)"

def parse_raw(rows, version):
    for row in rows:
        cols = row.find_all('td')
        if version != "beatmania IIDX 25 CANNON BALLERS" and len(cols) == 1:
            if(re.match("beatmania", cols[0].text)):
                version = cols[0].text
                if(version.endswith(" ▲ ▼ △")):
                    version = version[:-6]
                if(version.endswith(" ▲ △") or version.endswith(" ▼ △")):
                    version = version[:-4]
        if len(cols) == 11:
            #basic
            #if it ends in leggendaria, there's only another difficulty
            #if it ends with hcn just nope
            title = cols[9].text
            artist = cols[10].text
            genre = cols[8].text
            bpm = cols[7].text
            if title.endswith(leggendaria_mark) or title.endswith(leggendaria):
                get_song(get_level(cols[3]), 4, version, "single", title, artist, genre, bpm)
                get_song(get_level(cols[6]), 4, version, "double", title, artist, genre, bpm)
            elif not title.endswith(hcn):
                #note to self: This really shows... my dedication for details. aka kinda unnecessary but why not for the sake of correctness here?
                if title.startswith('crew['):
                    get_song(get_level(cols[1]), 1, version, "single", "crew -original mix-", "beatnation Records feat.星野奏子", "ANTHEM", bpm)
                    get_song(get_level(cols[2]), 2, version, "single", "crew -VENUS mix-", "beatnation Records feat.VENUS", "WITHOUT YOU TONIGHT", bpm)
                    get_song(get_level(cols[3]), 3, version, "single", "crew -Ryu☆ mix-", "beatnation Records feat.NU-KO", "EURODANCE", bpm)
                    get_song(get_level(cols[4]), 1, version, "double", "crew -original mix-", "beatnation Records feat.星野奏子", "ANTHEM", bpm)
                    get_song(get_level(cols[5]), 2, version, "double", "crew -kors k mix-", "beatnation Records feat.星野奏子", "J-CORE", bpm)
                    get_song(get_level(cols[6]), 3, version, "double", "crew -Prim version-", "beatnation Records feat.Prim", "Hi ANTHEM", bpm)
                elif title.startswith('Evans['):
                    get_song(get_level(cols[0]), 0, version, "single", "Evans -prototype-", artist, genre, bpm)
                    get_song(get_level(cols[1]), 1, version, "single", "Evans -prototype-", artist, genre, bpm)
                    get_song(get_level(cols[2]), 2, version, "single", "Evans -prototype-", artist, genre, bpm)
                    get_song(get_level(cols[3]), 3, version, "single", "Evans", artist, genre, bpm)
                    get_song(get_level(cols[4]), 1, version, "double", "Evans -prototype-", artist, genre, bpm)
                    get_song(get_level(cols[5]), 2, version, "double", "Evans -prototype-", artist, genre, bpm)
                    get_song(get_level(cols[6]), 3, version, "double", "Evans", artist, genre, bpm)
                elif artist.startswith('DJ SWAN['):
                    get_song(get_level(cols[0]), 0, version, "single", title, "DJ SWAN", genre, bpm)
                    get_song(get_level(cols[1]), 1, version, "single", title, "DJ SWAN", genre, bpm)
                    get_song(get_level(cols[2]), 2, version, "single", title, "DJ SWAN", genre, bpm)
                    get_song(get_level(cols[3]), 3, version, "single", title, "DJ SWAN (Toshiaki Komiya & Keiichi Ueno)", genre, bpm)
                    get_song(get_level(cols[4]), 1, version, "double", title, "DJ SWAN", genre, bpm)
                    get_song(get_level(cols[5]), 2, version, "double", title, "DJ SWAN", genre, bpm)
                    get_song(get_level(cols[6]), 3, version, "double", title, "DJ SWAN (Toshiaki Komiya & Keiichi Ueno)", genre, bpm)
                else:
                    get_song(get_level(cols[0]), 0, version, "single", title, artist, genre, bpm)
                    get_song(get_level(cols[1]), 1, version, "single", title, artist, genre, bpm)
                    get_song(get_level(cols[2]), 2, version, "single", title, artist, genre, bpm)
                    get_song(get_level(cols[3]), 3, version, "single", title, artist, genre, bpm)
                    get_song(get_level(cols[4]), 1, version, "double", title, artist, genre, bpm)
                    get_song(get_level(cols[5]), 2, version, "double", title, artist, genre, bpm)
                    get_song(get_level(cols[6]), 3, version, "double", title, artist, genre, bpm)
    return

parse_raw(cb_old_rows, "")
parse_raw(cb_new_rows, "beatmania IIDX 25 CANNON BALLERS")

print ("Writing json")

final_data = {
    "id": "beatmaniaiidx25",
    "songs": songs
}
#remember to change the datetime to the one from the page, not on the date it was update on the app's server
with open('../games/iidx/25/' +  version_name + '.json', 'w') as file:
    json.dump(final_data, file, indent=2, sort_keys=True)
print ("Finished")

data = {}
with open('../games/game_data.json') as file:
    data = json.load(file)
    data["games"]["iidx"]["versions"]["25"]["current"] = version_name
    array = data["games"]["iidx"]["versions"]["25"]["builds"]
    check = False
    for x in array:
        if(version_name == x):
            check = True
    if not check:
        data["games"]["iidx"]["versions"]["25"]["builds"].append(version_name)
print ("Finished reading game_data.json file")

with open('../games/game_data.json', 'w') as file:
    json.dump(data, file, indent=2, sort_keys=True)
print ("Finished updating game_data.json file")


"""
Simple CRF exp with basic features (lemma & POS in a 5-token window),
training on train and dev data combined and testing on test data.
"""

from os.path import join

from sklearn_crfsuite import CRF

from os.path import join, abspath, dirname
import sys
sys.path.append(abspath(join(dirname(__file__), '../..')))

from sie import ENTITIES, LOCAL_DIR
from sie.feats import generate_feats, features1
from sie.exp import run_exp_test, eval_exp_train

# Step 1: Generate features

train_spacy_dir = join(LOCAL_DIR, 'train', 'spacy')
train_base_feats_dir = join('_train', 'features1')

# If you want to save time by reusing existing feats, comment out the line below:
generate_feats(train_spacy_dir, train_base_feats_dir, features1)

dev_spacy_dir = join(LOCAL_DIR, 'dev', 'spacy')
dev_base_feats_dir = join('_dev', 'features1')

# If you want to save time by reusing existing feats, comment out the line below:
generate_feats(dev_spacy_dir, dev_base_feats_dir, features1)

test_spacy_dir = join(LOCAL_DIR, 'test', 'spacy')
test_base_feats_dir = join('_test', 'features1')

generate_feats(test_spacy_dir, test_base_feats_dir, features1)

# Step 2: Run experiments

crf = CRF(c1=0.1, c2=0.1, all_possible_transitions=True)
train_feat_dirs = [train_base_feats_dir]
dev_feat_dirs = [dev_base_feats_dir]
test_feat_dirs = [test_base_feats_dir]
preds = {}

for label in ENTITIES:
    preds[label] = run_exp_test(crf, train_feat_dirs, dev_feat_dirs, test_feat_dirs, label)

# Step 3: Evaluate

# Even though test data is unlabeled, but generates teh Brat files to submit
eval_exp_train(preds, 'test')


import buffer
from tkinter import *

def main():
	cl = buffer.Commandline(buffer.buffers, ("Roboto Mono", 10))
	start_buffer = buffer.FileBuffer("main.py", "main.py", ("Roboto Mono", 10))
	buffer.root.mainloop()

if __name__ == '__main__':
	main()


class Movie():
    """Note:
        Do not include the `self` parameter in the ``Args`` section.

    Args:
        title (str): Movie title.
        storyline (str): Movie storyline.
        image (str): Movie Image URL.
        trailer (str): Youtube movie trailer URL.

    Attributes:
        title (str): Movie title.
        storyline (str): Movie storyline.
        poster_image_url (str): Movie Image URL.
        trailer_youtube_url (str): Youtube movie trailer URL.

    """

    def __init__(self, title, storyline, image, trailer):
        self.title = title
        self.storyline = storyline
        self.poster_image_url = image
        self.trailer_youtube_url = trailer


from scoring import user_scoring_function
from models.user import User
from models.story import Story
from models.read import Read
from mongoengine import connect
from display_helper import *

def recommend(user_id):
    user = User.objects.get(user_id=user_id)
    feed = [] # users could have a attribute feed in which we store the feed and refresh it daily or hourly.
    for story in Story.objects:
        score = user_scoring_function(user, story)
        if score:
            feed.append((score, story.title))
    feed.sort(reverse=True)
    display_feed(feed, user)
    return feed

def recommend_test(user_id):
    # used to perform a first test of the system
    user = User.objects.get(user_id=user_id)
    feed = []

    for story_title in user.stories:
        story = Story.objects.get(title=story_title)
        score = user_scoring_function(user, story)
        if score:
            real_score = user.stories[story_title] - user.avg_speed
            feed.append((score, story.title, real_score))

    feed.sort(reverse=True)
    display_feed_test(feed, user)
    return feed

if __name__ == '__main__':
    connect('pocketgems')
    recommend_test(12) # test on user 12


class Bitmap(object):
	def __init__(self, max):
		self.size  = self.calcElemIndex(max, True)
		self.array = [0 for i in range(self.size)]

	def calcElemIndex(self, num, up=False):
		'''up为True则为向上取整, 否则为向下取整'''
		if up:
			return int((num + 31 - 1) / 31) #向上取整
		return int(num / 31)

	def calcBitIndex(self, num):
		return num % 31

	def set(self, num):
		elemIndex = self.calcElemIndex(num)
		byteIndex = self.calcBitIndex(num)
		elem      = self.array[elemIndex]
		self.array[elemIndex] = elem | (1 << byteIndex)

	def clean(self, i):
		elemIndex = self.calcElemIndex(i)
		byteIndex = self.calcBitIndex(i)
		elem      = self.array[elemIndex]
		self.array[elemIndex] = elem & (~(1 << byteIndex))

	def test(self, i):
		elemIndex = self.calcElemIndex(i)
		byteIndex = self.calcBitIndex(i)
		if self.array[elemIndex] & (1 << byteIndex):
			return True
		return False

class HashBitMap( object ):
    def __init__(self, max_size:int, len = 8 ):
        self.size = max_size

def FindSumCombination(arr:list, diff:int):
    """
    1. unsigned int ?
    2. exclude self ?
    3.
    :param arr:
    :param diff:
    :return:
    """
    dc = { }
    cb = { }
    for index, value in enumerate(arr):
        if not dc.get( value ):
            dc[ value ] = [ ]
        # end if
        # could be duplicated index
        dc[ value ].append( index )

    for key in dc.keys():
        other = diff - key
        if dc.get( other ) != None:
           first_index = dc.get( key )
           first_index_str = ",".join( [str(x) for x in first_index] )
           sec_index = dc.get( other )
           if first_index == sec_index:
               # ignore self
               continue
           sec_index_str = ",".join( [ str( x ) for x in sec_index ] )
           all_values = cb.values()
           if first_index_str not in all_values:
                cb[ first_index_str ] = sec_index_str

    for item in cb.items():
        print("comb is {0}".format( item ))

def printPairSums( data, sum ):

    first = 0
    size = len( data )
    last = size -1
    s = 0
    while (first < last):
        s = data[last] - data[first]
        if (s == sum):
            print("first index is {0}, last index is {1}".format( first, last ))
            first = first + 1
            last = first - 1
        elif (s < sum):
            first = first+1
        else:
            last= last-1

def topmaxheap( arr:list, k:int):
    pass

def testhashsum():
    arr = [1,2,3,4,4,5]
    d = 6
    FindSumCombination(arr, d)

if __name__ == "__main__":
    MAX = 879
    suffle_array = [ 45, 2, 78, 35, 67, 90, 879, 0, 340, 123, 46 ]
    result = [ ]
    bitmap = Bitmap( MAX )
    bitmap.clean(0)
    for num in suffle_array :
        bitmap.set( num )

    for i in range( MAX + 1 ) :
        print(i)
        if bitmap.test( i ) :
            result.append( i )

    print( '原始数组为:    %s' % suffle_array )
    print( '排序后的数组为: %s' % result )


'''
Created on Sep 30, 2014

@author: eturner
'''
from PE03.prod.PrimeFactor import isPrime

def nthPrime(n):
    i = 0
    primeNum = 0
    trial = 2
    while i < n:
        if isPrime(trial):
            i += 1
            primeNum = trial
        trial += 1

    return primeNum


from my_recognizer import recognize
from asl_utils import show_errors
from my_model_selectors import *
import numpy as np
import pandas as pd
from asl_data import AslDb
from matplotlib import (cm, pyplot as plt, mlab)
from asl_data import SinglesData
from itertools import product

asl = AslDb()  # initializes the database

### Ground Features
features_ground = ['grnd-rx', 'grnd-ry', 'grnd-lx', 'grnd-ly']
asl.df['grnd-ry'] = asl.df['right-y'] - asl.df['nose-y']
asl.df['grnd-rx'] = asl.df['right-x'] - asl.df['nose-x']
asl.df['grnd-ly'] = asl.df['left-y'] - asl.df['nose-y']
asl.df['grnd-lx'] = asl.df['left-x'] - asl.df['nose-x']

### Polar Features
features_polar = ['polar-rr', 'polar-rtheta', 'polar-lr', 'polar-ltheta']

def calculate_polar_radius(x, y):
    return (asl.df[x].pow(2) + asl.df[y].pow(2)).pow(0.5)

def calculate_polar_theta(x, y):
    return np.arctan2(asl.df[x], asl.df[y])

asl.df[features_polar[0]] = calculate_polar_radius('grnd-rx', 'grnd-ry')
asl.df[features_polar[1]] = calculate_polar_theta('grnd-rx', 'grnd-ry')
asl.df[features_polar[2]] = calculate_polar_radius('grnd-lx', 'grnd-ly')
asl.df[features_polar[3]] = calculate_polar_theta('grnd-lx', 'grnd-ly')

### Delta Features
features_delta = ['delta-rx', 'delta-ry', 'delta-lx', 'delta-ly']

def calculate_diff(column):
    return asl.df[column].diff().fillna(0)

asl.df[features_delta[0]] = calculate_diff('right-x')
asl.df[features_delta[1]] = calculate_diff('right-y')
asl.df[features_delta[2]] = calculate_diff('left-x')
asl.df[features_delta[3]] = calculate_diff('left-y')

### Norm Features
features_norm = ['norm-rx', 'norm-ry', 'norm-lx', 'norm-ly']
df_means = asl.df.groupby('speaker').mean()
df_std = asl.df.groupby('speaker').std()

def calculate_norm(coordinate):
    speaker = asl.df['speaker']
    return (asl.df[coordinate] - (speaker.map(df_means[coordinate]))) / (speaker.map(df_std[coordinate]))

asl.df[features_norm[0]] = calculate_norm('right-x')
asl.df[features_norm[1]] = calculate_norm('right-y')
asl.df[features_norm[2]] = calculate_norm('left-x')
asl.df[features_norm[3]] = calculate_norm('left-y')

## Ground Norm-Feature
features_ground_norm = ['grnd-norm-rx', 'grnd-norm-ry', 'grnd-norm-lx', 'grnd-norm-ly']
asl.df[features_ground_norm[0]] = calculate_norm(features_ground[0])
asl.df[features_ground_norm[1]] = calculate_norm(features_ground[1])
asl.df[features_ground_norm[2]] = calculate_norm(features_ground[2])
asl.df[features_ground_norm[3]] = calculate_norm(features_ground[3])

## Polar Norm-Feature
features_polar_norm = ['polar-norm-rr', features_polar[1], 'polar-norm-lr', features_polar[3]]
asl.df[features_polar_norm[0]] = calculate_norm(features_polar[0])
asl.df[features_polar_norm[2]] = calculate_norm(features_polar[2])

def calculate_error_rate(guesses: list, test_set: SinglesData):
    """ Print WER and sentence differences in tabular form

    :param guesses: list of test item answers, ordered
    :param test_set: SinglesData object
    :return:
        nothing returned, prints error report

    WER = (S+I+D)/N  but we have no insertions or deletions for isolated words so WER = S/N
    """
    S = 0
    N = len(test_set.wordlist)
    num_test_words = len(test_set.wordlist)
    if len(guesses) != num_test_words:
        print("Size of guesses must equal number of test words ({})!".format(num_test_words))
    for word_id in range(num_test_words):
        if guesses[word_id] != test_set.wordlist[word_id]:
            S += 1

    return float(S) / float(N)

def train_all_words(features, model_selector):
    training = asl.build_training(features)  # Experiment here with different feature sets defined in part 1
    sequences = training.get_all_sequences()
    Xlengths = training.get_all_Xlengths()
    model_dict = {}
    for word in training.words:
        model = model_selector(sequences, Xlengths, word,
                               n_constant=3).select()
        if model:
            model_dict[word] = model
    return model_dict

# TODO Choose a feature set and model selector
def recognize_with(features, model_selector):
    models = train_all_words(features, model_selector)
    test_set = asl.build_test(features)
    probabilities, guesses = recognize(models, test_set)
    return guesses, test_set

def recognize_with_combination(selector_and_features):
    features = selector_and_features[1]
    selector = selector_and_features[0]
    guesses, test_set = recognize_with(features[1], selector[1])
    return calculate_error_rate(guesses, test_set)

def print_to_grid(results):
    [print(f"{result[0][0]}\t|\t{result[1][0]}\t|\t{result[2]}") for result in results]

selectors = {"cv": SelectorCV, "bic": SelectorBIC, "dic": SelectorDIC}
features = {"ground": features_ground, "norm": features_norm, "delta": features_delta, "polar": features_polar, "grnd-norm": features_ground_norm, "polar-norm": features_polar_norm}
nr_runs = len(selectors) * len(features)

combinations = [(s, f) for s in selectors.items() for f in features.items()]
results = map(lambda c: (c[0], c[1], recognize_with_combination(c)), combinations)

print("\n\nBEGIN Result")
print_to_grid(results)
print("\n\nEND Result")


import csv
import operator

from TrendingTopics.DBLocation import DBLocation

def get_prepared_woeid_map(file):
    """creates a list with {utc_offet: set(woeids)}"""
    locations = get_woeid_map(file)

    prep = dict()

    for l in locations:
        woeid = l.woeid
        tmp = {}
        for l2 in locations:
            if woeid == l2.woeid:
                utc = l2.utc_offset
                if utc not in tmp:
                    tmp[utc] = 1
                else:
                    tmp[utc] += 1
        utc_offset = max(tmp.items(), key=operator.itemgetter(1))[0]
        if utc_offset not in prep:
            prep[utc_offset] = set()
            if woeid is not None:
                prep[utc_offset].add(woeid)
        else:
            if woeid is not None:
                prep[utc_offset].add(woeid)
    return prep

def get_woeid_map(file):
    woeids = []
    f = open(file, 'rt', encoding="ISO-8859-1")
    reader = csv.reader(f, delimiter=';')
    for row in reader:
        loc = DBLocation()
        loc.name = row[0]
        loc.utc_offset = int(row[1])
        if row[2] != 'null':
            loc.woeid = int(row[2])

        woeids.append(loc)

    f.close()
    return woeids


#!/usr/bin/env python
# -*- coding: utf-8 -*-

# from __future__ import print_function, unicode_literals, absolute_import

# 171. Excel Sheet Column Number
# Given a column title as appear in an Excel sheet, return its corresponding column number.

# 总结：
# relates to: p168, p172

class Solution(object):

    def titleToNumber(self, s):
        """
        :type s: str
        :rtype: int
        """
        return reduce(
            lambda x, y: x * 26 + y,
            map(lambda x: ord(x) - ord('A') + 1, s)
        )

import unittest

class TestSolution(unittest.TestCase):

    def test_demo(self):
        self.assertEqual(1, 1)
        self.assertTrue(True)

suite = unittest.TestLoader().loadTestsFromTestCase(TestSolution)
unittest.TextTestRunner(verbosity=2).run(suite)

if __name__ == '__main__':
    print('ok')


from underthehood import *

def setreg(wo,coor):
    #specifies outer boundary, counter clockwisely
    j=coor
    r=[]
    for i in range(1,len(j)):
        r.append([j[i-1][0],j[i-1][1],j[i][0],j[i][1]])
    for i in range(0,len(r)):
        if r[i][0]==r[i][2]:
            #vert
            if r[i][3]>r[i][1]:
                #up
                wo.zones.append(zone(r[i][0],r[i][3],r[i][0]+10,r[i][1],2))
            else:
                #down
                wo.zones.append(zone(r[i][0]-10,r[i][1],r[i][0],r[i][3],3))
        else:
            #horiz
            if r[i][2]>r[i][0]:
                #goes to right
                wo.zones.append(zone(r[i][0],r[i][1]-10,r[i][2],r[i][3],1))
            else:
                #goes to left
                wo.zones.append(zone(r[i][2],r[i][3],r[i][0],r[i][1]+10,0))

class worldlist:
    wlist=[]
    active=0
    def g(self):
        return self.wlist[self.active]
    def s(self,a):
        self.active=a
    def __init__(self):
        #world0
        self.wlist.append(world("basic"))
        setreg(self.wlist[0],[[100,100],[100,400],[600,400],[600,100],[100,100]])
        self.wlist[0].blocks.append(block(100,100,600,400,WHITE))
        #world1
        self.wlist.append(world("secondary"))
        setreg(self.wlist[1],[[10,10],[10,180],[180,180],[180,10],[10,10]])
        setreg(self.wlist[1],[[40,40],[150,40],[150,150],[40,150],[40,40]])
        self.wlist[1].blocks.append(block(10,10,180,180,WHITE))
        self.wlist[1].blocks.append(block(40,40,150,150,BLACK))


'''
Given a non-empty array of integers, every element appears twice except for one. Find that single one.

Note:

Your algorithm should have a linear runtime complexity. Could you implement it without using extra memory?

Example 1:

Input: [2,2,1]
Output: 1
Example 2:

Input: [4,1,2,1,2]
Output: 4
'''

class Solution(object):
    def singleNumber(self, nums):
        """
        :type nums: List[int]
        :rtype: int
        """
        # if len(nums) == 1:
        #     return nums[0]

        # count_arr = [0] * len(nums)

        # for i in nums:
        #     count_arr[i] += 1
        # print(count_arr)
        # return [count_arr.index(i) for i in count_arr if i == 1][0]

        dic = dict()
        dic2 = dict()
        for i in nums:
            dic[i] = 0
        for i in nums:
            dic[i] += 1
        for i, j in dic.items():
            dic2[j] = i
        return dic2.get(1)

print(Solution.singleNumber(0, [4,1,2,1,2]))
print(Solution.singleNumber(0, [1]))
print(Solution.singleNumber(0, [-1,-1,-2]))

[i for i in range(3) if i ==1][0]


from __future__ import absolute_import, division, print_function
import logging
import scapy.config
import scapy.layers.l2
import scapy.route
import socket
import math
import errno

logging.basicConfig(format='%(asctime)s %(levelname)-5s %(message)s', datefmt='%Y-%m-%d %H:%M:%S', filename='../logs/ipscan.log', level=logging.DEBUG)
logger = logging.getLogger(__name__)

def long2net(arg):
    if (arg <= 0 or arg >= 0xFFFFFFFF):
        raise ValueError("illegal netmask value", hex(arg))
    return 32 - int(round(math.log(0xFFFFFFFF - arg, 2)))

def mac2int(mac):
    return int(mac.replace(':',''),16)

def to_CIDR_notation(bytes_network, bytes_netmask):
    network = scapy.utils.ltoa(bytes_network)
    netmask = long2net(bytes_netmask)
    net = "%s/%s" % (network, netmask)
    if netmask < 16:
        logger.warn("%s is too big. skipping" % net)
        return None

    return net

def scan_and_find_mac(net, interface, target_mac, timeout=1):
    logger.info("Start arping %s on %s" % (net, interface))
    try:
        ans, unans = scapy.layers.l2.arping(net, iface=interface, timeout=timeout, verbose=False)
        logger.info("Found %s responsive IPs and %s non-responsive IPs." % (len(ans), len(unans)))

        if len(ans)>0:
            for s, r in ans.res:
                devMAC = r.sprintf("%Ether.src%")
                devIP = r.sprintf("%ARP.psrc%")
                #logger.info("Network device " + devMAC + " at " + devIP)

                if mac2int(devMAC) == mac2int(target_mac):
                    logger.info("Bingo! Target device " + devMAC + " found at " + devIP)
                    try:
                        socket.inet_aton(devIP)
                        logger.info("IP " + devIP + " is valid.")
                        return devIP
                    except socket.error:
                        logger.warn("IP " + devIP + " is not valid.")
        else:
            logger.error("No responsive IPs found; %s on %s" % (net, interface))
        return ''

    except socket.error as e:
        if e.errno == errno.EPERM:     # Operation not permitted
            logger.error("%s. Did you run as root?", e.strerror)
        else:
            raise

def find_mac_on_network(mac):
    for network, netmask, _, interface, address in scapy.config.conf.route.routes:

        # skip loopback network and default gw
        if network == 0 or interface == 'lo' or address == '127.0.0.1' or address == '0.0.0.0':
            continue

        if netmask <= 0 or netmask == 0xFFFFFFFF:
            continue

        net = to_CIDR_notation(network, netmask)

        if interface != scapy.config.conf.iface:
            # see http://trac.secdev.org/scapy/ticket/537
            logger.warn("skipping %s because scapy currently doesn't support arping on non-primary network interfaces", net)
            continue

        if net:
            return scan_and_find_mac(net, interface, mac)


import board    #hardware details of the specific board that you are using
import digitalio
import time

led = digitalio.DigitalInOut(board.D13)
led.direction = digitalio.Direction.OUTPUT

while True:
    led.value = True
    time.sleep(1.0)
    led.value = False
    time.sleep(1.0)


from django.shortcuts import render
from django.template import loader,Context
from django.http import HttpResponse
from . import models  #leave this as this always dont change it
from django.http import *
from django.shortcuts import redirect

# Create your views here.
def user_list(request):
    users=models.user_database.objects.all()
    template=loader.get_template('expense/user_list.html')
    context= {
            'users':users ,
    }
    return HttpResponse(template.render(context,request))

def expence_list(request,user_name):
    user=models.user_database.objects.get(user_name=user_name)
    id=user.user_id
    expense_list=models.user_expense.objects.filter(user_id=id)
    context={
        'user':user,
        'expense_list':expense_list
    }
    template=loader.get_template('expense/user_expense_list.html')
    return HttpResponse(template.render(context,request))


import numpy as np
import sklearn
from sklearn import preprocessing
from SplitData import split_data
from matplotlib import pyplot as plt

def normalizer(imf_index,plot=False):
    ts, ts_train, ts_valid, ts_test, cpu_values, cpu_train, cpu_valid, cpu_test, \
            ram_values, ram_train, ram_valid, ram_test=split_data(imf_index,plot=False)

    std_cpu = np.std(cpu_values)
    std_ram = np.std(ram_values)
    mean_cpu = np.mean(cpu_values)
    mean_ram = np.mean(ram_values)
    print('means are : ',mean_ram,mean_cpu)
    print('Stds are : ', std_ram, std_cpu)
    print('*****************')

    #cpu_values_normalize = (np.array(cpu_values) - mean_cpu) / std_cpu
    max_abs_scaler = preprocessing.MaxAbsScaler()
    cpu_values_normalize = max_abs_scaler.fit_transform(cpu_values.reshape(-1, 1))
    # cpu_values_normalize = sklearn.preprocessing.normalize(cpu_values.reshape(-1, 1))
    #cpu_values_normalize=cpu_values_normalize/np.max(np.abs(cpu_values_normalize))

    #ram_values_normalize = (np.array(ram_values) - mean_ram) / std_ram
    ram_values_normalize = max_abs_scaler.fit_transform(ram_values.reshape(-1, 1))
    # ram_values_normalize = sklearn.preprocessing.normalize(ram_values.reshape(-1, 1))
    #ram_values_normalize = ram_values_normalize / np.max(np.abs(ram_values_normalize))

    # '''--------------------------  Reload Data --------------------------------------- '''
    # desired_len = 100
    # ts_reload,cpu_reloaded_normalize,ram_reloaded_normalize=\
    #         Reload_Data_RF.Reload_Data_RF(ts,cpu_values_normalize,ram_values_normalize,desired_len)
    #
    #
    # print('length of original data is ', len(cpu_values))
    # print('length of Reloaded Data is ',len(cpu_reloaded_normalize),len(ram_reloaded_normalize))
    # print('---------------------------------------------------')
    #
    # if plot:
    #     plt.subplot(2, 1, 1)
    #     plt.plot(ts, cpu_values_normalize, color='red', label='cpu-original-data')
    #     plt.ylabel('CPU Req normalized')
    #     plt.xlabel('Time symbol')
    #     plt.legend()
    #     plt.subplot(2, 1, 2)
    #     plt.plot(ts_reload, cpu_reloaded_normalize, color='blue', label='cpu-Reloaded-data')
    #     plt.ylabel('CPU Req normalized')
    #     plt.legend()
    #     plt.xlabel('Time symbol')
    #     plt.show()
    #
    #     plt.subplot(2, 1, 1)
    #     plt.plot(ts, ram_values_normalize, color='red', label='RAM-original-data')
    #     plt.ylabel('CPU Req normalized')
    #     plt.xlabel('Time symbol')
    #     plt.legend()
    #     plt.subplot(2, 1, 2)
    #     plt.plot(ts_reload, ram_reloaded_normalize, color='blue', label='RAM-Reloaded-data')
    #     plt.ylabel('CPU Req normalized')
    #     plt.legend()
    #     plt.xlabel('Time symbol')
    #     plt.show()

    # return std_cpu,std_ram,mean_cpu,mean_ram,ts,ts_reload, \
    #        cpu_values_normalize,cpu_reloaded_normalize, \
    #                 ram_values_normalize,ram_reloaded_normalize

    return std_cpu,std_ram,mean_cpu,mean_ram,ts,cpu_values_normalize,ram_values_normalize