# Filename: 10_train_gridsearch.py
# Function: script te train the network
# Remark: Initial source based on Tensorflow v1 usage
# Version 2 20191117 changed for tensorflow 2.0

# To run tensorflow for cpu and Ubuntu Linux platform
# you can use prebuild tensorflow package.
# You have to manually build the tensorflow package.
# Prebuild tenosrflow package on Linux
# gives a runtime warning about CPU extentions AVX2, FMA.
# See article https://stackoverflow.com/questions/47068709/your-cpu-supports-instructions-that-this-tensorflow-binary-was-not-compiled-to-u
#
# Build a TensorFlow pip package from source and install it on Ubuntu Linux and macOS
# See https://www.tensorflow.org/install/source

# See https://docs.bazel.build/versions/master/install.html
# See https://docs.bazel.build/versions/master/install-ubuntu.html#install-with-installer-ubuntu
# See http://bazel.build/docs/getting-started.html to start a new project!

# Install required packages before install eand run bazel
# run script pip_install_before_bazel_install.sh
# ./before_bazel_install/pip_install_before_bazel_install.sh

# used commands to install:
# download bazel install script # see https://docs.bazel.build/versions/master/install-ubuntu.html#install-with-installer-ubuntu
# install bazel install script with
# chmod +x bazel-<version>-installer-linux-x86_64.sh
# ./bazel-<version>-installer-linux-x86_64.sh --user
# cd <project directory>
# without a git respository
# git clone https://github.com/tensorflow/tensorflow.git
# with a git repository
# git submodule add https://github.com/tensorflow/tensorflow.git
# cd tensorflow
# for release 2.0 tensorflow
# git checkout r2.0
# See https://www.tensorflow.org/install/source for options asked when ./configure is ran.
# To "specify optimization flags" use --config=mkl to use mkl-dnn library build with ./mkl-dnn-setup/setup.sh
# ./configure
# build tensorflow package. Beware to use the correct command depending on with version
# bazel build //tensorflow/tools/pip_package:build_pip_package --local_ram_resources=8192
# create wheel file in ~/Documents/sources/python/python3/python3_Muziek_Generator/MLMG/tensorflow_pkg
# run build_pip_package from ./tensorflow directory
# bazel-bin/tensorflow/tools/pip_package/build_pip_package ~/Documents/sources/python/python3/python3_Muziek_Generator/MLMG/tensorflow_pkg
# Install the package tensorflow-2.0.0-cp36-cp36m-linux_x86_64.whl
# pip install ~/Documents/sources/python/python3/python3_Muziek_Generator/MLMG/tensorflow_pkg/tensorflow-2.0.0-cp36-cp36m-linux_x86_64.whl

# Info after successful build of release r2.0
# INFO: Elapsed time: 11160.249s, Critical Path: 239.71s
# INFO: 17804 processes: 17804 local.
# INFO: Build completed successfully, 18538 total actions

""" This module prepares midi file data and feeds it to the neural
    network for training """

import glob
import pickle

import numpy

from music21 import chord, converter, instrument, note
import tensorflow as tf

from sklearn.model_selection import GridSearchCV
from sklearn.exceptions import FitFailedWarning
# https://scikit-learn.org/stable/tutorial/basic/tutorial.html
# https://www.guru99.com/scikit-learn-tutorial.html
# https://scikit-learn.org/stable/tutorial/machine_learning_map/index.html

import warnings

# from keras.callbacks import ModelCheckpoint # tensorflow v1
# from keras.layers import LSTM, Activation, Dense, Dropout # tensorflow v1
# from keras.models import Sequential  # tensorflow v1
# from keras.utils import utils # tensorflow v1. this does not work use tf.keras.utils.<function> as call

# homeDir when this script is used from my Laptop
homeDir = '/home/claude/Documents/sources/python/python3/python3_Muziek_Generator/MLMG/'
# homeDir old laptop
#homeDir = '/home/claude/Documents/sources/python/python3/cx1964ReposMusic21MLMG/'
# homeDir when this script is used from my Virtualbox Linux VM
#homeDir = '/home/test/Documents/sources/python/python3/cx1964ReposMusic21MLMG/'

def train_network():
    """ Train a Neural Network to generate music """

    print("train_network() begin")

    # get_notes()
    # see p5 of article
    # read midi files which gives a list of notes
    notes = get_notes()

    # get amount of unqiue pitch names
    n_vocab = len(set(notes))
    #print("n_vocab = amount of unique pitch names:", n_vocab)
    #print("len notes: ", len(notes))

    # prepare_sequences()
    # see p6 of article
    # map notes to numerical data
    # ToDo: study code from here
    network_input, network_output = prepare_sequences(notes, n_vocab)
    print("train_network() step1: call create_network()")
    model = create_network(network_input, n_vocab)

    #train(model, network_input, network_output) : original code
    #train(model, network_input, network_output, n_vocab)
    print("train_network() step2: call train()")
    train(model=model, network_input=network_input, network_output=network_output, network_n_vocab=n_vocab)

def get_notes():
    """ Get all the notes and chords from the midi files in the ./midi_songs directory """

    print("get_notes() begin")
    notes = []

    for file in glob.glob(homeDir+"midi_songs/*.mid"):
        midi = converter.parse(file)

        #print("Parsing %s" % file)

        notes_to_parse = None

        try: # file has instrument parts
            s2 = instrument.partitionByInstrument(midi)
            notes_to_parse = s2.parts[0].recurse()
        except: # file has notes in a flat structure
            notes_to_parse = midi.flat.notes

        for element in notes_to_parse:
            if isinstance(element, note.Note):
                notes.append(str(element.pitch))
            elif isinstance(element, chord.Chord):
                notes.append('.'.join(str(n) for n in element.normalOrder))

    #print("notes:", notes)

    with open(homeDir+'data/notes', 'wb') as filepath:
        pickle.dump(notes, filepath)

    return notes # return een list of notes. Offset informatie (=tijd) gaat verloren

def prepare_sequences(notes, n_vocab):
    """ Prepare the sequences used by the Neural Network """
    sequence_length = 100 # voorspel de volgende noot obv 100 voorgaande noten
    # See for anology textGenLSTM.py
    # See https://machinelearningmastery.com/text-generation-lstm-recurrent-neural-networks-python-keras/

    # get all unique pitch names
    pitchnames = sorted(set(item for item in notes))
    #print("pitchnames: ", pitchnames)
    # create a dictionary to map pitches to integers
    #        for number, note in enumerate(pitchnames) genereert een reeks met elementen inde vorm <rangnummer>, <pitchname>
    note_to_int = dict((note, number) for number, note in enumerate(pitchnames))
    #print("note_to_int", str(note_to_int))

    network_input = []
    network_output = []

    # create input sequences and the corresponding outputs
    for i in range(0, len(notes) - sequence_length, 1):
        sequence_in = notes[i:i + sequence_length]
        sequence_out = notes[i + sequence_length]

        # hier worden list met notes mapped naar een list met integers mbv gebruikt van de note_to_int dict
        # mapping is nodig omdat neural netwerk met integers werkt om de gewichten uit te kunnen rekenen.
        network_input.append([note_to_int[char] for char in sequence_in])

        # hier worden list met notes mapped naar een list met integers mbv gebruikt van de note_to_int dict
        # mapping is nodig omdat neural netwerk met integers werkt om de gewichten uit te kunnen rekenen.
        network_output.append(note_to_int[sequence_out])

    n_patterns = len(network_input)

    # reshape the input into a format compatible with LSTM layers
    network_input = numpy.reshape(network_input, (n_patterns, sequence_length, 1))

    # normalize input
    network_input = network_input / float(n_vocab)

    #print("network_output:",network_output )
    # Converts a class vector (integers) to binary class matrix
    # See https://www.tensorflow.org/api_docs/python/tf/keras/utils
    #network_output = utils.to_categorical(network_output) # tbv tf 1
    network_output = tf.keras.utils.to_categorical(network_output) # tf v2

    return (network_input, network_output) # return input and output list with mapped notes

# ToDo
def create_network(ntwrk_input, ntwrk_n_vocab):
    # Create wrapper for create_network because of KerasClassifier create_network needs parameter ntwrk_n_vocab
    # https://stackoverflow.com/questions/40393629/how-to-pass-a-parameter-to-scikit-learn-keras-model-function
    # See also https://machinelearningmastery.com/grid-search-hyperparameters-deep-learning-models-python-keras/

    # This Function is used to create model, required for KerasClassifier
    # This function must be transformed to somethink like  def create_model(optimizer= rmsprop , init= glorot_uniform ):
    # p60 pdf Jason Brownlee Deep Learning with python
    # paragraph 9.3 Grid Search Deep Learning Model Parameters

    """ create the structure of the neural network """
    # Zie orginele definitie create_network
    # https://towardsdatascience.com/how-to-generate-music-using-a-lstm-neural-network-in-keras-68786834d4c5
    # https://github.com/Skuldur/Classical-Piano-Composer

    # zie alternatief https://adgefficiency.com/tf2-lstm-hidden/

    # Uit artikel mbt input_shape in onderstaande tf.keras.layers.LSTM(
    # For the first layer we have to provide a
    # unique parameter called input_shape. The purpose of the parameter is
    # to inform the network of the shape of the data it will be training.
    # Geldt ook voor tensorflow v2 ????

    # model = Sequential() # tensorflow v1

    # issues "Skipping optimization due to error while loading function libraries: Invalid argument: Functions"
    #        see https://github.com/tensorflow/tensorflow/issues/30263
    #        work arround in all tf.keras.layers.LSTM calls
    #        change param activation from tf.nn.tanh to None
    print("create_network() begin")
    #print("This script has Still an issue: ")
    #print("Skipping optimization due to error while loading function libraries: Invalid argument: Functions")
    #print("https://github.com/tensorflow/tensorflow/issues/30263")

    network_input = ntwrk_input
    network_n_vocab = ntwrk_n_vocab
    def model():
        # create model():
        #print("model():")
        nn = tf.keras.models.Sequential([  # tensorflow v2
                tf.keras.layers.LSTM(
                  # 512, orgineel tf v1
                  512 # aantal nodes in layer uit artikel v1; Geldit ook voor v2?
                 ,input_shape=(network_input.shape[1], network_input.shape[2]) # zie artikel. Geldt dit ik ook voor v2?
                                                                           # first layer need this parameter
                 ,return_sequences=True # also tensorflow v2 LSTM argument
                 ,activation=tf.nn.tanh# see issue. Use explicitly default value tanh
                )
               ,tf.keras.layers.Dropout(0.3)
               ,tf.keras.layers.LSTM( 512
                                     ,return_sequences=True
                                     ,activation=tf.nn.tanh # see issue. Use explicitly default value tanh
                                    )
               ,tf.keras.layers.Dropout(0.3)
               ,tf.keras.layers.LSTM( 512
                                     ,activation=tf.nn.tanh # see issue. Use explicitly default value tanh

                                    )
               ,tf.keras.layers.Dense(256) # For tf 2.0
                                           # activation: Activation function to use.
                                           # If you don't specify anything,
                                           # no activation is applied (ie. "linear" activation: a(x) = x).
                                           # check if this also valid voor tf 1.0
               ,tf.keras.layers.Dropout(0.3)
               ,tf.keras.layers.Dense( network_n_vocab # what does n_vocab mean ????
                                      ,activation=tf.nn.softmax
                                     )
                #tf.keras.layers.Activation('softmax') # This is move to previous line
             ])

        #model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
        nn.compile( optimizer=tf.keras.optimizers.RMSprop()  # Optimizer
                   ,loss=tf.keras.losses.CategoricalCrossentropy() # Loss function to minimize
                   ,metrics=['accuracy'] # added
                  )
        #print("Na compile")

        # show used model
        nn.summary()
        return nn

    return model

def train(model, network_input, network_output, network_n_vocab):
    """ train the neural network """

    # See pdf Jason Brownlee Deep Learning with python
    # paragraph 9.3 Grid Search Deep Learning Model Parameters

    print("train() begin")

    # Wraper
    # Truc for passwing parameters to create_network()
    def cr_network():
        print("... cr_network()")
        print("... from cr_network() call create_network()")
        create_network(network_input,network_n_vocab)

    # create_network call with 2 parameters
    # https://scikit-learn.org/stable/tutorial/machine_learning_map/index.html
    print("train() call KerasClassifier")
    model = tf.keras.wrappers.scikit_learn.KerasClassifier( build_fn=cr_network
                                                           ,verbose=0)
    #print("na tf.keras.wrappers.scikit_learn.KerasClassifier")
    # grid search epochs, batch size and optimizer
    optimizers = [ 'rmsprop' , 'adam' ]
    init = [ 'normal' , 'uniform' ] # 'glorot_uniform' is deprecated see https://machinelearningmastery.com/use-keras-deep-learning-models-scikit-learn-python/
    epochs = numpy.array([50, 100, 150])
    batches = numpy.array([5, 10, 20])
    param_grid = dict(nb_epoch=epochs, batch_size=batches) #, init=init, optimizer=optimizers)
    #print("train()  voor GridSearchCV()")

    #warnings.simplefilter('always', FitFailedWarning)
    grid = GridSearchCV( estimator=model
                        ,param_grid=param_grid
                        ,verbose=10 # setting verbose = 10 will print the progress for every 10 task completion
                        ,n_jobs=-1 # -1, the process will use all cores on your machine
                        ,cv=5) # Cross validation is used to evaluate each individual model and the default of 3-fold cross validation is used,
                               # although this can be overridden by specifying the cv argument

    print("train()  na GridSearchCV()")
    X=network_input
    y=network_output
    #print("train() type(X):",type(X)," type(y):",type(y))
    print("train()  voor grid.fit()")

    #ToDo
    # Probleem zit in grid.fit
    # Als grid_result = grid.fit(X=X, y=y) uitzet, daan geen fout melding
    grid_result = grid.fit(X=X, y=y)
    '''
    with warnings.catch_warnings(record=True) as w:
         try:
             grid_result = grid.fit(X=X, y=y)
         except ValueError:
             pass
         print("opgevangen fout: ", repr(w[-1].message))
    '''

    print("train()  na grid.fit()")
    # summarize results

    '''
    print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))
    for params, mean_score, scores in grid_result.grid_scores_:
        print("%f (%f) with: %r" % (scores.mean(), scores.std(), params))
    '''

    # methode2: creeer een hdf5 file
    # Zie pagina 95 - 96 van pdf (14.3 Checkpoint best Neural Network Model only)
    # De file waarin tgv het checkpoint process tijdens de leerfase de weights in worden weggeschreven.
    filepath = "weights-best.hdf5"

    # Zie ook paragraaf 14.2 en 14.3

    # het leer proces
    #print("na model.fit")

if __name__ == '__main__':

    # tf 2.0
    #print("tf.version.VERSION: ", tf.version.VERSION)
    #print("tf.version.GIT_VERSION: ", tf.version.GIT_VERSION)

    #print("Eager execution running: ", tf.executing_eagerly())
    print("keras version: ", tf.keras.__version__)

    print("\n\n__main__ call train_network()")
    train_network()


__author__ = 'yuhenglong@google.com (Yuheng Long)'

import multiprocessing
import unittest

from generation import Generation
from mock_task import IdentifierMockTask
import pipeline_process
import steering

# Pick an integer at random.
STEERING_TEST_STAGE = -8

# The number of generations to be used to do the testing.
NUMBER_OF_GENERATIONS = 20

# The number of tasks to be put in each generation to be tested.
NUMBER_OF_TASKS = 20

# The stride of permutation used to shuffle the input list of tasks. Should be
# relatively prime with NUMBER_OF_TASKS.
STRIDE = 7

class MockGeneration(Generation):
  """This class emulates an actual generation.

  It will output the next_generations when the method Next is called. The
  next_generations is initiated when the MockGeneration instance is constructed.
  """

  def __init__(self, tasks, next_generations):
    """Set up the next generations for this task.

    Args:
      tasks: A set of tasks to be run.
      next_generations: A list of generations as the next generation of the
        current generation.
    """
    Generation.__init__(self, tasks, None)
    self._next_generations = next_generations

  def Next(self, _):
    return self._next_generations

  def IsImproved(self):
    if self._next_generations:
      return True
    return False

class SteeringTest(unittest.TestCase):
  """This class test the steering method.

  The steering algorithm should return if there is no new task in the initial
  generation. The steering algorithm should send all the tasks to the next stage
  and should terminate once there is no pending generation. A generation is
  pending if it contains pending task. A task is pending if its (test) result
  is not ready.
  """

  def testSteering(self):
    """Test that the steering algorithm processes all the tasks properly.

    Test that the steering algorithm sends all the tasks to the next stage. Test
    that the steering algorithm terminates once all the tasks have been
    processed, i.e., the results for the tasks are all ready.
    """

    # A list of generations used to test the steering stage.
    generations = []

    task_index = 0
    previous_generations = None

    # Generate a sequence of generations to be tested. Each generation will
    # output the next generation in reverse order of the list when the "Next"
    # method is called.
    for _ in range(NUMBER_OF_GENERATIONS):
      # Use a consecutive sequence of numbers as identifiers for the set of
      # tasks put into a generation.
      test_ranges = range(task_index, task_index + NUMBER_OF_TASKS)
      tasks = [IdentifierMockTask(STEERING_TEST_STAGE, t) for t in test_ranges]
      steering_tasks = set(tasks)

      # Let the previous generation as the offspring generation of the current
      # generation.
      current_generation = MockGeneration(steering_tasks, previous_generations)
      generations.insert(0, current_generation)
      previous_generations = [current_generation]

      task_index += NUMBER_OF_TASKS

    # If there is no generation at all, the unittest returns right away.
    if not current_generation:
      return

    # Set up the input and result queue for the steering method.
    manager = multiprocessing.Manager()
    input_queue = manager.Queue()
    result_queue = manager.Queue()

    steering_process = multiprocessing.Process(
        target=steering.Steering,
        args=(set(), [current_generation], input_queue, result_queue))
    steering_process.start()

    # Test that each generation is processed properly. I.e., the generations are
    # processed in order.
    while generations:
      generation = generations.pop(0)
      tasks = [task for task in generation.Pool()]

      # Test that all the tasks are processed once and only once.
      while tasks:
        task = result_queue.get()

        assert task in tasks
        tasks.remove(task)

        input_queue.put(task)

    task = result_queue.get()

    # Test that the steering algorithm returns properly after processing all
    # the generations.
    assert task == pipeline_process.POISONPILL

    steering_process.join()

  def testCache(self):
    """The steering algorithm returns immediately if there is no new tasks.

    If all the new tasks have been cached before, the steering algorithm does
    not have to execute these tasks again and thus can terminate right away.
    """

    # Put a set of tasks in the cache and add this set to initial generation.
    test_ranges = range(NUMBER_OF_TASKS)
    tasks = [IdentifierMockTask(STEERING_TEST_STAGE, t) for t in test_ranges]
    steering_tasks = set(tasks)

    current_generation = MockGeneration(steering_tasks, None)

    # Set up the input and result queue for the steering method.
    manager = multiprocessing.Manager()
    input_queue = manager.Queue()
    result_queue = manager.Queue()

    steering_process = multiprocessing.Process(
        target=steering.Steering,
        args=(steering_tasks, [current_generation], input_queue, result_queue))

    steering_process.start()

    # Test that the steering method returns right away.
    assert result_queue.get() == pipeline_process.POISONPILL
    steering_process.join()

if __name__ == '__main__':
  unittest.main()


import glob
import cv2
import numpy as np
import tensorflow as tf
from operator import itemgetter
import difflib
import os
import shutil
from shutil import copyfile
MIN_CONTOUR_AREA = 10
lst = glob.glob('*.png')
l_img = []
for i in range(len(lst)):
    l_ROI = []
    img = cv2.imread(lst[i],0)
# inverting pixels of image to make contours
    img_thresh = 255 - img
#plotting contours
    img_contours,n_contours,n_hierarchy = cv2.findContours(img_thresh,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE)
    for j in range(len(n_contours)):
        if cv2.contourArea(n_contours[j])>MIN_CONTOUR_AREA:
            [intX, intY, intW, intH] = cv2.boundingRect(n_contours[j])
            cv2.rectangle(img,(intX, intY),(intX+intW,intY+intH),(255, 255, 255),1)
            imgROI = img[intY:intY+intH, intX:intX+intW]
            l_ROI.append([intX,imgROI])
# l _img contains all the segregated images of each image in a document along with the x co-ordinate of top right corner
# l_img[i] gives list of all segregated characters of ith image of a document  along with it x co-ordinate
# l_img[i][1][0] gives matrix of first character of ith image in a document
    l_img.append(l_ROI)
l_sorted = []
for i in range(len(l_img)):
    temp = sorted(l_img[i],key=itemgetter(0))
    l_sorted.append(temp)
#l_sorted list contains sorted images according to there top right x co-ordinate
lt = []
#padding of images in the list with zeros to make the size 100X100
for i in range(len(l_sorted)):
    for j in range(len(l_img[i])):
        l_sorted[i][j][1] = 255 - l_sorted[i][j][1]
        l_sorted[i][j][1] = cv2.resize(l_sorted[i][j][1],(30,50))
        npad = ((25,25),(35,35))
        l_sorted[i][j][1] = np.pad(l_sorted[i][j][1], pad_width=npad, mode='constant', constant_values=0)
l_image_doc = []
for i in range(len(l_sorted)):
    l_image = []
    for j in range(len(l_sorted[i])):
        l_sorted[i][j][1] = np.reshape(l_sorted[i][j][1],(100,100,1))
        l_image.append(l_sorted[i][j][1])
    l_image_doc.append(l_image)
l_array = []
for i in range(len(l_image_doc)):
    X = np.asarray(l_image_doc[i])
    X_image = np.reshape(X,(X.shape[0],100*100))
    l_array.append(X_image)
# l_array contains list of all the images in a document converted into array of size (number of characterX10000)
def Weights(shape):
    initial = tf.truncated_normal(shape,stddev = 0.1)
    return tf.Variable(initial)
def bias(shape):
    initial = tf.constant(0.1,shape=shape)
    return tf.Variable(initial)
x = tf.placeholder(tf.float32,[None,10000])
y_ = tf.placeholder(tf.float32,[None,22])
keep_prob = tf.placeholder(tf.float32)
x_image = tf.reshape(x,[-1,100,100,1])
def conv(x,W,s_c):
    return tf.nn.conv2d(x,W,strides=[1,s_c,s_c,1],padding ='VALID')
def max_pool(x,s_p,k_s):
    return tf.nn.max_pool(x,ksize=[1,k_s,k_s,1],strides=[1,s_p,s_p,1],padding='SAME')
def mean(l):
    return sum(l)/len(l)
#conv1_1
w_conv1_1 = Weights([11,11,1,96])
b_conv1_1 = bias([96])
h_conv1_1 = tf.nn.relu(conv(x_image,w_conv1_1,3)+b_conv1_1)
# maxpool
h_pool1 = max_pool(h_conv1_1,2,3)
#conv1_2
w_conv1_2 = Weights([5,5,96,256])
b_conv1_2 = bias([256])
h_conv1_2 = tf.nn.relu(conv(h_pool1,w_conv1_2,1)+b_conv1_2)
#maxpool
h_pool2 = max_pool(h_conv1_2,2,3)
    #conv2_1
w_conv2_1 = Weights([3,3,256,256])
b_conv2_1 = bias([256])
h_conv2_1 = tf.nn.relu(conv(h_pool2,w_conv2_1,1)+b_conv2_1)
#maxpool
h_pool3 = max_pool(h_conv2_1,2,3)
    #fully connected layer
w_fc1 = Weights([2*2*256,1024])
b_fc1 = bias([1024])
h_pool3_flat = tf.reshape(h_pool3, [-1, 2*2*256])
h_fc1 = tf.nn.relu(tf.matmul(h_pool3_flat, w_fc1) + b_fc1)
    #dropout
h_fc1_drop = tf.nn.dropout(h_fc1,keep_prob)
    #softmax layer
w_fc2 = Weights([1024,22])
b_fc2 = bias([22])
y_conv = tf.matmul(h_fc1_drop,w_fc2)+b_fc2
y1 = tf.nn.softmax(y_conv)
cost_function = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv))
train_step = tf.train.AdamOptimizer(0.0001).minimize(cost_function)
#correct_prediction = tf.equal(tf.argmax(y1,1), tf.argmax(y_,1))
prediction = tf.argmax(y_conv,1)
#accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
#a = np.matrix(h_conv1_1)
saver = tf.train.Saver()
init = tf.global_variables_initializer()
with tf.Session() as sess:
    sess.run(init)
    ac = []
    saver = tf.train.import_meta_graph('model_Alphabet-9.meta')
    sess.run(tf.global_variables_initializer())
    #tf.train.Saver.restore(sess=sess,save_path=')
    saver.restore(sess,tf.train.latest_checkpoint('.\\'))
    l_str = []
    for i in range(len((l_array))):
        str1 = []
        for j in range(l_array[i].shape[0]):
            #x_batch = np.reshape(l_array[i][j,:],(1,10000))
            l = (sess.run(prediction,{x:l_array[i],keep_prob:0.5}))
            if l[j] == 0:
                str1.append('A')
            if l[j] == 1:
                str1.append('a')
            if l[j] == 2:
                str1.append('B')
            if l[j] == 3:
                str1.append('c')
            if l[j] == 4:
                str1.append('d')
            if l[j] == 5:
                str1.append('e')
            if l[j] == 6:
                str1.append('E')
            if l[j] == 7:
                str1.append('f')
            if l[j] == 8:
                str1.append('H')
            if l[j] == 9:
                str1.append('I')
            if l[j] == 10:
                str1.append('m')
            if l[j] == 11:
                str1.append('n')
            if l[j] == 12:
                str1.append('N')
            if l[j] == 13:
                str1.append('o')
            if l[j] == 14:
                str1.append('P')
            if l[j] == 15:
                str1.append('R')
            if l[j] == 16:
                str1.append('r')
            if l[j] == 17:
                str1.append('S')
            if l[j] == 18:
                str1.append('t')
            if l[j] == 19:
                str1.append('u')
            if l[j] == 20:
                str1.append('v')
            if l[j] == 12:
                str1.append('y')
        s = ("".join(str1))
        l_str.append(s)
    #str2 = 'test'
    str1 = ['beneficiarysnameandaddress','panno','commodity',
'remmitersnameandaddress','valuedate','valuedateapplicableonlyifvaluedateintheswift','iecode', 'hscode']
    #seq=difflib.SequenceMatcher(None, str2,str1[i])
    for i in range(len(l_str)):
        d_max=0
        for j in range(8):
            a = l_str[i]
            str2 = a.lower()
            #print(str2)
            seq=difflib.SequenceMatcher(None, str2,str1[j])
            d=seq.ratio()*100
            #print(d)
            if d>d_max:
                d_max = d
                index = j
        #print(d_max)
        if d_max>70:
            if index == 0:
                print('beneficiary\'s name and address' )
                if not os.path.exists('beneficiary'):
                    os.mkdir('beneficiary')
                shutil.copy(lst[i],'beneficiary')
            if index == 1:
                print('PAN No')

                if not os.path.exists('pan'):
                    os.mkdir('pan')
                shutil.copy(lst[i],'pan')
            if index == 2:
                print('Commodity')
                if not os.path.exists('commodity'):
                    os.mkdir('commodity')
                shutil.copy(lst[i],'commodity')
            if index==3:
                print('Remmiter\'s name and address')
                if not os.path.exists('remmiter'):
                    os.mkdir('remmiter')
                shutil.copy(lst[i],'remmiter')
            #if index==4:
             #   print('Purpose of remmitence')
            if index == 4 or index == 5:
                print('value date')
                if not os.path.exists('value date'):
                    os.mkdir('value date')
                shutil.copy(lst[i],'value date')
            if index==6:
                print('I.E.Code')
                if not os.path.exists('I.E.Code'):
                    os.mkdir('I.E.Code')
                shutil.copy(lst[i],'I.E.Code')
            if index==7:
                print('H.S.Code')
                if not os.path.exists('H S Code'):
                    os.mkdir('H S Code')
                shutil.copy(lst[i],'H S Code')
        else:
            if not os.path.exists('garbage'):
                os.mkdir('garbage')
            shutil.copy(lst[i],'garbage')


import json
from statistics import mean
import requests
import pprint
import sys
import time

def km_to_mile(kilometers: float) -> float:
    return kilometers * 0.62137

def avg_velocity(url) -> float:
    print('Starting Avg Velocity Calculator . . .\nGathering Data (approx. 10 sec) . . .')
    velocities = []
    for x in range (0, 5):
        response = requests.get(url)
        if response.status_code == 200:
            data = json.loads(response.text)
            velocity = data['velocity']
            velocities.append(velocity)
        else:
            sys.stderr.write(f'Error: {response.status_code}')
            exit(1)
        time.sleep(2)
    return mean(velocities)

if __name__ == "__main__":
    print('Hello and Welcome to JSON Parsing with Python!\n')

    url = 'https://api.wheretheiss.at/v1/satellites/25544'
    response = requests.get(url)

    if response.status_code == 200:
        data = json.loads(response.text)
        pprint.pprint(data)
        name = data['name']
        timestamp = data['timestamp']
        altitude = data['altitude']
        velocity = data['velocity']
        print(f'\nInformation in (km) for the {name}\n        time:    {timestamp}\n    Altitude: {altitude} (km)\n    Velocity: {velocity} (km/hr)\n')
        print('Lets convert to miles!\n')
        altitude = km_to_mile(altitude)
        velocity = km_to_mile(velocity)
        print(f'Information in (mi) for the {name}\n        time:    {timestamp}\n    Altitude: {altitude} (mi)\n    Velocity: {velocity} (mi/hr)\n')
    else:
        sys.stderr.write(f'Error: {response.status_code}')
        exit(1)

    #avg_velocity = avg_velocity(url)
    #print(f'ISS Average Velocity (km/hr): {avg_velocity}')
    #print(f'ISS Average Velocity (mi/hr): {km_to_mile(avg_velocity)}')


import cv2
import numpy as np

img = cv2.imread("assets/wire.png")
img1 = np.zeros((300, 300, 3), np.uint8)
img1.fill(200)
dirs = [(0,1), (1,0), (0,-1), (-1,0)]
count, i, pos_x, pos_y = 200, 0, -1, 0
dis = count//2
while dis > 0:
    for dir in dirs:
        dis = count//2
        for _ in range(dis):
            pos_x = pos_x + dir[0]
            pos_y = pos_y + dir[1]
            img1[pos_x, pos_y] = img[0, i]
            i += 1
        count -= 1
cv2.imshow("result", img1)
cv2.waitKey(0)


import collections
import math
import os

def path(filename):
  filepath = os.path.realpath(__file__)
  dirpath = os.path.dirname(filepath)
  return os.path.join(dirpath, filename)

def line(a, b, x, y):
  import turtle
  turtle.up()
  turtle.goto(a, b)
  turtle.down()
  turtle.goto(x, y)

class Vector(collections.Sequence):
  PRECISION = 6
  __slots__ = ('_x', '_y', '_hash')

  def __init__(self, x, y):
    self._hash = None
    self._x = round(x, self.PRECISION)
    self._y = round(y, self.PRECISION)

  @property
  def x(self):
    return self._x

  @x.setter
  def x(self, value):
    if self._hash is not None:
      raise ValueError('Cannot set x after hashing')
    self._x = round(value, self.PRECISION)

  @property
  def y(self):
    return self._y

  @y.setter
  def y(self, value):
    if self._hash is not None:
      raise ValueError('Cannot set y after hashing')
    self._y = round(value, self.PRECISION)

  def __hash__(self):
    if self._hash is None:
      pair = (self.x, self.y)
      self._hash = hash(pair)
    return self._hash

  def __len__(self):
    return 2

  def __getitem__(self, index):
    if index == 0:
      return self.x
    elif index == 1:
      return self.y
    else:
      raise IndexError

  def copy(self):
    return type(self)(self.x, self.y)

  def __eq__(self, other):
    if isinstance(other, Vector):
      return self.x == other.x and self.y == other.y
    return NotImplemented

  def __ne__(self, other):
    if isinstance(other, Vector):
      return self.x != other.x and self.y != other.y
    return NotImplemented

  def __iadd__(self, other):
    if self._hash is not None:
      raise ValueError('Cannot add vector after hashing')
    elif isinstance(other, Vector):
      self.x += other.x
      self.y += other.y
    else:
      self.x += other
      self.y += other
    return self

  def __add__(self, other):
    copy = self.copy()
    return copy.__iadd__(other)

  __radd__ = __add__

  def move(self, other):
    self.__iadd__(other)

  def __isub__(self, other):
    if self._hash is not None:
      raise ValueError('Cannot subtract vector after hashing')
    elif isinstance(other, Vector):
      self.x -= other.x
      self.y -= other.y
    else:
      self.x -= other
      self.y -= other
    return self

  def __sub__(self, other):
    copy = self.copy()
    return copy.__isub__(other)

  def __imul__(self, other):
    if self._hash is not None:
      raise ValueError('Cannot multiply vector after hashing')
    elif isinstance(other, Vector):
      self.x *= other.x
      self.y *= other.y
    else:
      self.x *= other
      self.y *= other
    return self

  def __mul__(self, other):
    copy = self.copy()
    return copy.__imul__(other)

  __rmul__ = __mul__

  def scale(self, other):
    self.__imul__(other)

  def __itruediv__(self, other):
    if self._hash is not None:
      raise ValueError('Cannot divide vector after hashing')
    elif isinstance(other, Vector):
      self.x /= other.x
      self.y /= other.y
    else:
      self.x /= other
      self.y /= other
    return self

  def __truediv__(self, other):
    copy = self.copy()
    return copy.__itruediv__(other)

  def __neg__(self):
    copy = self.copy()
    copy.x = -copy.x
    copy.y = -copy.y
    return copy

  def __abs__(self):
    return (self.x**2 + self.y**2)**0.5

  def rotate(self, angle):
    if self._hash is not None:
      raise ValueError('Cannot rotate vector after hashing')
    radians = angle*math.pi/180.0
    cosine = math.cos(radians)
    sine = math.sin(radians)
    x = self.x
    y = self.y
    self.x = x*cosine - y*sine
    self.y = y*cosine + x*sine

  def __repr__(self):
    name = type(self).__name__
    x = self.x
    y = self.y
    return f'{name} ({x!r}, {y!r})'

Python Game Development/Games with Turtle/base.py


"""
This module provides TUI functions and prompts.
"""
import questionary
from questionary import Choice, Style, Separator
from questionary.prompts.common import InquirerControl

custom_style = Style([
    ('question',    '#af00d7'),                     # question text
    ('instruction', '#0000ff'),                     # user instructions for select, rawselect, checkbox
    ('disabled',    'fg:#858585 italic'),           # disabled choices for select and checkbox prompts
    ('answer',      "bold fg:#223377"),             # submitted answer text behind the question
    ('pointer',     'fg:#673ab7 bold'),             # pointer used in select and checkbox prompts
    ('selected',    'fg:#cc5454'),                  # style for a selected item of a checkbox
    ('separator',   'fg:#cc5454 bold italic'),      # separator in lists

    ("normal1",     "bold #0000ff"),
    ("normal2",     "bold #af00d7"),
    ("warning1",    "bold #875f87"),
    ("warning2",    "bold red"),
    ("exists",      "bold #5fd700")
])

def generate_choices(choices_names: list[str], default_choice: int=None, values: list=None, shortcuts: list[str]=None) -> list[Choice]:
    """
    Description:
        Generate a list of `Choice` objects.
    ---
    Parameters:
        `choices_names` (`list[str]`)
            A list containing the names of the choices.

        `default_choice` (`int`)
            An index to a default choice from the provided `choices_names`.

        `values` (`list`)
            The values of the choices that will be returned when selected. Defaults to `choices_names`.

        `shortcuts` (`list[str]`)
            A list of one character to be used as shortcuts for the choices.
            Defaults to a list of number from `1` to `len(choices_names)`.
    ---
    Returns: `list[Choice]`.
    """
    return [Choice(title=[("class:normal2", choice_name)], checked=default_choice == i,
                   value=None if not values else values[i], shortcut_key=str(i+1) if not shortcuts else shortcuts[i])
            for i, choice_name in enumerate(choices_names)]

def issue_selection_question(message: str, choices_names: list[str], default_choice=0, values: list=None, shortcuts: list=None, shortcuts_instruction_msg: str=None):
    """
    Description:
        Prompt the user to select a choice and return its corresponding value.
    ---
    Parameters:
        `message` (`str`)
            A custom message to display as the prompt question.

        `choices_names` (`list[str]`)
            A list containing the names of the choices.

        `default_choice` (`int`)
            An index to a default choice from the provided `choices_names`.

        `values` (`list`)
            The values of the choices that will be returned when selected. Defaults to `choices_names`.

        `shortcuts` (`list[str]`)
            A list of one character to be used as shortcuts for the choices.
            Defaults to a list of number from `1` to `len(choices_names)`.

        `shortcuts_instruction_msg` (`str`)
            A message to display after the prompt question (for control hints and shortcuts).
    ---
    Returns:
        ("Any") => The corresponding value to the selected choice.
    """
    generated_choices = generate_choices(choices_names, default_choice, values, shortcuts)
    if not shortcuts_instruction_msg:
        shortcuts_instruction_msg = "(Use up/down arrows and number keys to navigate)"
    else:
        shortcuts_instruction_msg = f"(Use up/down arrow keys and {shortcuts_instruction_msg} buttons to navigate)"

    output = questionary.select(message, choices=generated_choices, style=custom_style, qmark="➥", pointer="➤",
                                default=generated_choices[default_choice], use_shortcuts=True,
                                show_selected=False, use_indicator=False, instruction=shortcuts_instruction_msg).ask()
    return generated_choices[default_choice].value if output is None else output

def issue_yes_no_question(message: str, default_choice=0, values=[0, 1], choices_names=["No", "Yes"], shortcuts=["n", "y"]):
    """
    Description:
        Prompt the user to select from two choices and return the corresponding value to the selected choice.
    ---
    Parameters:
        `message` (`str`)
            A custom message to display as the prompt question.

        `default_choice` (`int`)
            An index to a default choice from the provided `choices_names`.

        `values` (`list`)
            The values of the choices that will be returned when selected. Defaults to `choices_names`.

        `choices_names` (`list[str]`)
            A list containing the names of the choices.

        `shortcuts` (`list[str]`)
            A list of one character to be used as shortcuts for the choices.
            Defaults to a list of number from `1` to `len(choices_names)`.
    ---
    Returns:
        ("Any") => The corresponding value to the selected choice.
    """
    shortcuts_help = ("UP/down" if not default_choice else "up/DOWN") + " arrows and "
    shortcuts_help += f"{str(shortcuts[0]).upper()}/{shortcuts[1]}" if not default_choice else f"{shortcuts[0]}/{str(shortcuts[1]).upper()}"
    return issue_selection_question(message, choices_names, default_choice, values, shortcuts, f"(Use {shortcuts_help} keys to navigate)")


import numpy as np
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import sys

plt.style.use('fivethirtyeight')

models = ['lcn', 'fcn1', 'fcn2', 'cnn']
qend="False"

prune_rates = [0, 0.001, 0.005, 0.01, 0.02, 0.03, 0.04, 0.05, 0.1, 0.15, 0.25, 0.5]
print(prune_rates)

for dorefa, bitw in [("True", 4), ("True", 8), ("True", 16), ("False", 32)]:
    errs = np.zeros((len(models), len(prune_rates)))
    sps = np.zeros((len(models), len(prune_rates)))
    for j, model in enumerate(models):
        logpath='/data/sls/u/meng/skanda/home/thesis/dorefa/real/'
        logpath+='pruned_models/{}_{}_32_{}_{}/'.format(model,bitw,qend,dorefa)
        logpath+='log.log'

        with open(logpath,'r') as f:
            for line in f:
                if 'Errors:[' in line:
                    st = line.split('Errors:')[1]
                    errs[j] = np.fromstring(st[1:-1], dtype=float, sep=', ')
                elif 'Sparsities:[' in line:
                    st = line.split('Sparsities:')[1]
                    sps[j] = np.fromstring(st[1:-1], dtype=float, sep=', ')

    points = np.array(errs)
    fig = plt.figure(figsize=(16,8))
    ax1 = fig.add_subplot(111)
    ax1.set_xlabel('Sparsity')
    ax1.set_ylabel('Speaker ID Error')
    for i, name in enumerate(['Locally Connected', 'Fully Connected (Small)', 'Fully Connected (Large)', 'Convolutional']):
        if i < points.shape[0]:
            ax1.plot(prune_rates, points[i,:], '-o', label=name)
        print(dorefa, name, bitw, errs[i])
    ax1.legend(loc=2)
    title = "Error vs. Induced Sparsity"
    if dorefa == "True":
        title += ", FxPt Quant. {}-Bit".format(bitw)
        filename="dorefa_{}".format(bitw)
    else:
        title += ", Full-Precision"
        filename="baseline"
    plt.title(title)
    plt.savefig('figs/pruned/%s.png' % (filename,))
    print(dorefa, bitw, sps[:,1])


import csv
import json

csvFilePath = 'death_natl.csv'
jsonFilePath = 'death_data.json'

def read_CSV(csvFilePath, jsonFilePath):
    csv_rows = []
    with open(csvFilePath) as csvfile:
        reader = csv.DictReader(csvfile)
        field = reader.fieldnames
        for row in reader:
            csv_rows.extend([{field[i]:row[field[i]] for i in range(len(field))}])
            convert_write_json(csv_rows, jsonFilePath)

#Convert csv data into json and format it to make it human readable
def convert_write_json(data, jsonFilePath):
    with open(jsonFilePath, "w") as f:
        f.write(json.dumps(data, sort_keys=False, indent=4, separators=(',', ': ')))

read_CSV(csvFilePath, jsonFilePath)


import os
import torch

import torchvision.transforms as T

from collections import Counter

from lib.dataset.audio import get_audio_x
from lib.dataset.video import  get_video_x

import time

class MmDataSet(torch.utils.data.Dataset):
    def __init__(self, input_file,  args ={}, mode_train_val = "training", n_sample= 1 ):

        self.input_file = input_file
        self.args = args
        self.n_sample = n_sample

        param_dataset = args["dataset"]
        self.path_frame_folders = f'{param_dataset["data_dir"]}/{param_dataset["dir_frames"]}'
        self.path_audio_folder = f'{param_dataset["data_dir"]}/{param_dataset["dir_audios"]}'

        self.mode_train_val = mode_train_val

        self.parse_input_file()

        #self.__getitem__(0)
        #exit()

    def __getitem__(self, index):

        record = self.records[index]
        x_video, x_audio = "None", "None"
        ID = record["ID"]
        y = int(record["label"])

        if self.args["model"]["video_segments"] > 0:
            x_video = torch.stack([get_video_x(record, self.args, self.mode_train_val) for _ in range(self.n_sample)])

        if self.args["model"]["audio_segments"] > 0:
            x_audio = torch.stack([get_audio_x(record, self.args, self.mode_train_val) for _ in range(self.n_sample)])

        return  ID,   y , x_video, x_audio

    def __len__(self):
        return len(self.records)

    def parse_input_file(self):
        self.records = []
        error_not_exist, error_no_frames, error_no_wav = {}, {}, {}
        stat_labels = Counter()

        lines = [x.strip().split(' ') for x in open(self.input_file)]

        for i, item in enumerate(lines):
            ID = item[0]
            label = int(item[1])

            indicator, ifolder, ifolder_size, wavefile = self.check(label, ID, error_not_exist, error_no_frames, error_no_wav)

            if indicator:
                line_data = {}
                line_data["imagefolder"] = ifolder
                line_data["imagefolder_size"] = ifolder_size
                line_data["audio_file"] = wavefile

                line_data["ID"] = ID
                line_data["label"] = label
                stat_labels[label] += 1

                self.records.append(line_data)

        print("stat labels", len(stat_labels))
        print("error_not_exist", len(error_not_exist))
        print("error_no_frames", len(error_no_frames))
        print("error_no_wav", len(error_no_wav))
        print("self.records:", len(self.records))

    def check(self, label, ID, error_not_exist, error_no_frames, error_no_wav):

        ifolder, ifolder_size, wavefile = "", "", ""

        """ Check folder with frames exist and has minimum number of frames"""

        if self.args["model"]["video_segments"] > 0:

            ifolder = f"{self.path_frame_folders}/{ID}"

            if not os.path.isdir(ifolder):
                error_not_exist[ID] = 1
                return False , ifolder , ifolder_size , wavefile

            ifolder_size = int(len(os.listdir(ifolder)))
            if ifolder_size < 3:
                error_no_frames[ID] = ifolder_size
                return False , ifolder , ifolder_size, wavefile

        """ Check wavefile exist """

        if self.args["model"]["audio_segments"] > 0:
            wavefile = f"{self.path_audio_folder}/{ID}.wav"
            if not os.path.isfile(wavefile):
                error_no_wav[ID] = 1
                return False , ifolder , ifolder_size, wavefile

        return True, ifolder, ifolder_size, wavefile

def GetDataSet(args, mode_train_val="training",n_sample= 1):

    param_dataset = args["dataset"]

    if mode_train_val == "training":
        file_list = f'{param_dataset["data_dir"]}/{param_dataset["file_train_list"]}'
    else:
        file_list = f'{param_dataset["data_dir"]}/{param_dataset["file_val_list"]}'

    MMD = MmDataSet(file_list, args=args, mode_train_val=mode_train_val,n_sample= n_sample )

    return MMD

def GetDataLoaders(MMD, args,  mode_train_val="training"):

    shuffle = False
    if mode_train_val == "training": shuffle = True

    data_loader = torch.utils.data.DataLoader(
        MMD,
        batch_size=args["training_param"]["batch_size"],
        num_workers=args["training_param"]["num_workers"],
        shuffle=shuffle, pin_memory=False, drop_last=False
    )

    return data_loader

def visualise_x(x, ID, folder):

    (n_batches, n_samples, n_segment, c, h, w) = x.size()

    for j_batch in range(n_batches):

        folder_id = f"{folder}/{ID[j_batch]}"
        if not os.path.exists(folder_id): os.mkdir(folder_id)

        for j_sample in range(n_samples):
            for j_seg in range(n_segment):
                file_ID = f"sample_{j_sample}_segment_{j_seg}"
                T.ToPILImage()(x[j_batch,j_sample,j_seg ]).save(f'{folder_id}/{file_ID}.png', mode='png')

def test_input( train_loader, model_input, device, folder):

    if not os.path.exists(folder): os.mkdir(folder)

    print(f'test_input, see output at: {folder}\n')

    count_IDs = 0
    for i, (ID, y, x_video, x_audio) in enumerate(train_loader):

        if x_video[0] != "None": print("x_video", x_video.size())
        if x_audio[0] != "None": print("x_audio", x_audio.size())

        if x_video[0] != "None": x_video = x_video.to(device)
        if x_audio[0] != "None": x_audio = x_audio.to(device)

        x = model_input(x_video, x_audio)

        visualise_x(x, ID, folder)
        count_IDs += len(ID)

        if count_IDs > 10: break


import numpy as np
import matplotlib.pyplot as plt
import scipy.stats as stats

# asset parameters
expected_return = 0.12
volatility = 0.20

# risk-free rate
risk_free_rate = 0.05

# standardize the risk-free rate
z = (risk_free_rate - expected_return) / volatility

# compute the probability of underperformance
prob_underperformance = stats.norm.cdf(z)

# Define the distribution
dist = stats.norm(loc=expected_return, scale=volatility)

# Generate values
values = np.linspace(expected_return - 3*volatility, expected_return + 3*volatility, 100)

# Generate pdf values
pdf_values = dist.pdf(values)

# Create the plot
plt.figure(figsize=(10,6))
plt.plot(values, pdf_values, lw=2)

# Customize the title and labels
plt.title('Distribution of Returns and Underperformance', fontsize=14)
plt.xlabel('Returns', fontsize=12)
plt.ylabel('Probability Density', fontsize=12)

# Customize the grid and spines
plt.grid(alpha=0.5)
plt.gca().spines['top'].set_visible(False)
plt.gca().spines['right'].set_visible(False)

# Shade the underperformance region
underperformance_values = np.linspace(expected_return - 3*volatility, risk_free_rate, 100)
underperformance_pdf = dist.pdf(underperformance_values)
plt.fill_between(underperformance_values, underperformance_pdf, color='red', alpha=0.3)

# Annotate the risk-free rate
plt.axvline(x=risk_free_rate, color='red', linestyle='--', lw=1.5)
plt.annotate(f'Risk-free rate: {risk_free_rate*100:.1f}%', (risk_free_rate, max(pdf_values)/2),
             textcoords="offset points", xytext=(-15,10), ha='center', fontsize=9, color='red',
             bbox=dict(facecolor='white', edgecolor='red', boxstyle='round,pad=0.2'))

# Annotate the probability of underperformance
plt.text(expected_return - 3*volatility, max(pdf_values)*0.9, f'Prob. of Underperformance: {prob_underperformance*100:.2f}%', fontsize=10, bbox=dict(facecolor='white', alpha=0.8))

# Annotate asset parameters
plt.text(expected_return - 3*volatility, max(pdf_values)*0.8, f'Expected Return: {expected_return*100:.2f}%\nVolatility: {volatility*100:.2f}%', fontsize=10, bbox=dict(facecolor='white', alpha=0.8))

plt.show()


import os
import requests
import shutil

from dataclasses import dataclass
from zipfile import ZipFile

@dataclass
class Configuration:
    link: str = "https://github.com/DeepSportRadar/player-reidentification-challenge/archive/refs/heads/master.zip"
    md5: str = '05715857791e2e88b2f11e4037fbec7d'
    path: str = "../data"

def download_zip(url, save_path, chunk_size=128):
    r = requests.get(url, stream=True)
    with open(save_path, 'wb') as fd:
        for chunk in r.iter_content(chunk_size=chunk_size):
            fd.write(chunk)

#----------------------------------------------------------------------------------------------------------------------#
# Config                                                                                #
#----------------------------------------------------------------------------------------------------------------------#
config = Configuration()

#----------------------------------------------------------------------------------------------------------------------#
# Custom Dataset Initialization                                                                                #
#----------------------------------------------------------------------------------------------------------------------#
if not os.path.exists(config.path):
    print("Custom dataset path is not found. Initializing...")
    os.makedirs(config.path)
    os.mkdir(os.path.join(config.path, "custom_dataset"))
    os.mkdir(os.path.join(config.path, "custom_dataset", "gallery"))
    os.mkdir(os.path.join(config.path, "custom_dataset", "query"))
    os.mkdir(os.path.join(config.path, "custom_dataset", "training"))
else:
    print("Custom dataset path is already established.")

#----------------------------------------------------------------------------------------------------------------------#
# Download                                                                                #
#----------------------------------------------------------------------------------------------------------------------#
if not os.path.isfile("{}/synergyreid_data.zip".format(config.path)):
    print("Downloading demo dataset...")

    download_zip(url=config.link,
                 save_path="{}/reid_challenge.zip".format(config.path),
                 chunk_size=128)

    path_in_zip = "player-reidentification-challenge-master/baseline/data/synergyreid/raw/synergyreid_data.zip"

    with ZipFile("{}/reid_challenge.zip".format(config.path)) as z:

        z.extract(path_in_zip,
                  config.path)

    shutil.move("{}/{}".format(config.path, path_in_zip), "{}/synergyreid_data.zip".format(config.path))

    shutil.rmtree("{}/player-reidentification-challenge-master".format(config.path))
    os.remove("{}/reid_challenge.zip".format(config.path))
else:
    print("Demo dataset is already downloaded. Moving to next step...")

#----------------------------------------------------------------------------------------------------------------------#
# Unzip & Clean Up                                                                                #
#----------------------------------------------------------------------------------------------------------------------#
zip_file = "{}/synergyreid_data.zip".format(config.path)

print("Extracting zip file...")
with ZipFile(zip_file) as z:
    z.extractall(path=config.path)

shutil.rmtree(f"{config.path}/__MACOSX")

print("Extraction Completed!\n")


import math
import numpy as np
import pandas as pd
from PIL import Image
from sklearn.cluster import KMeans
from scipy.spatial import distance
import ntpath

class ImagesRepoModel:
    """A class to store different views of the image repository"""

    """ arbitrarily chosen global image size, to allow for easy image comparison """
    IMG_WIDTH = 320
    IMG_HEIGHT = 240

    def __validate_image(self, image_path):
        valid_formats = ('png', 'jpg', 'jpeg', 'tiff', 'bmp', 'gif')
        if not image_path.lower().endswith(valid_formats):
            raise ValueError
        img = Image.open(image_path)
        img.verify()

    def __compute_image_hashes(self, flattened_image_pixels):
        """ Bitwise XOR based hash of the RGB image pixels """

        img_hash = 0
        i = 0
        while i < self.IMG_WIDTH * self.IMG_HEIGHT * 3:
            """ compute sum of a chunk of pixels"""
            chunk_sum = 0
            for j in range(self.IMG_WIDTH):
                chunk_sum = chunk_sum + flattened_image_pixels[i + j]
            """ bitwise XOR between chunk sums"""
            img_hash = img_hash ^ chunk_sum
            i = i + self.IMG_WIDTH
        return img_hash

    def __define_image_space(self, pic):
        """ Account for comparing greyscale, coloured, and multi-sized images
            by normalizing all pictures to be RGB formatted with the same size. """

        img = Image.open(pic).convert('RGB')
        img = img.resize((self.IMG_WIDTH, self.IMG_HEIGHT))

        """ flatten pixel values """
        img_converted = np.array(img)
        pixel_row = img_converted.ravel()
        pixel_row_as_list = np.array(pixel_row.tolist())

        """ compute hash of image pixels """
        img_hash = self.__compute_image_hashes(pixel_row_as_list)

        return pixel_row_as_list, img_hash

    def __init__(self, image_folder=None, parent=None):
        """ Initialize the image repository using images in image_folder.
            Store names of image to allow for textual searching, and cluster the
            current images to allow for image-based searching """

        self.image_names = []
        self.parent = parent

        for image in image_folder:
            try:
                self.__validate_image(image)
                self.image_names.append(image)
            except (IOError, SyntaxError, ValueError):
                print('Bad image file, not including in repository: ', image)

        self.images_hash = []

        """ Construct matrix where there is a row per image, and columns representing pixel values.
            Note that there is 3*IMG_WIDTH*IMG_HEIGHT columns, since each pixel contributes 3 values (RGB) """
        self.df = pd.DataFrame(index=self.image_names, columns=np.arange(3 * self.IMG_WIDTH * self.IMG_HEIGHT))

        for pic in self.image_names:
            pixel_features, img_hash = self.__define_image_space(pic)
            self.df.loc[pic] = pixel_features
            self.images_hash.append([pic, img_hash])

        """ Cluster images such that they can be searched against """
        self.kmeans = KMeans(n_clusters=math.ceil(math.sqrt(len(self.image_names)))).fit(self.df)

    def text_search(self, query_string):
        """ Finds all images in the repository that have query_string as a substring of their filenmae"""

        print("Searching image repository based on inputted text query: " + query_string)

        search_results = list(filter(lambda k: query_string in ntpath.basename(k), self.image_names))
        if not search_results:
            print("No search results found")
            return self
        print("Found some hits! Now constructing new image repository view with your results...")

        filtered_repo = ImagesRepoModel(search_results, self)
        print(str(len(search_results)) + " Search Result(s): ")
        print(*filtered_repo.image_names, sep="\n")

        print("You are now in a view of the repository containing only the above search results.")
        return filtered_repo

    def image_search(self, query_image):
        """ Search for similar pictures to the image at path query_image,
        by finding images in the precomputed (KMeans) cluster corresponding to the
        centroid with the minimum euclidean distance to the query_image pixel vector"""

        print("Searching image repository for similar images based on query image: " + query_image)

        try:
            self.__validate_image(query_image)
        except (IOError, SyntaxError, ValueError):
            print('Inputted image is not valid: ', query_image)
            return self
        centroids = self.kmeans.cluster_centers_
        labels = self.kmeans.labels_

        pixel_features, img_hash = self.__define_image_space(query_image)

        min_distance = float('inf')
        cluster_belongs = -1
        for index, centroid in enumerate(centroids):
            dist_to_centroid = distance.euclidean(centroid, pixel_features)
            if dist_to_centroid < min_distance:
                min_distance = dist_to_centroid
                cluster_belongs = index

        search_results = []
        for index, name in enumerate(self.image_names):
            if labels[index] == cluster_belongs:
                search_results.append(name)

        if not search_results:
            print("No search results found")
            return self
        print("Found some hits! Now constructing new image repository view with your results...")

        filtered_repo = ImagesRepoModel(search_results, self)
        print(str(len(search_results)) + " Search Result(s): ")
        print(*filtered_repo.image_names, sep="\n")

        print("You are now in a view of the repository containing only the above search results.")
        return filtered_repo

    def check_for_corruption(self):
        """ Compare current and original images hashes to determine
         if any images in the repository have been modified """

        for img_path_and_hash in self.images_hash:
            orig_img_path = img_path_and_hash[0]
            orig_img_hash = img_path_and_hash[1]
            current_pixel_features, current_img_hash = self.__define_image_space(orig_img_path)
            if orig_img_hash != current_img_hash:
                print("Corrupted image! " + orig_img_path)


from dobot_api import dobot_api_dashboard, dobot_api_feedback, MyType
from multiprocessing import Process
from threading import Thread
import numpy as np
import time
import paho.mqtt.client as mqtt
import json
import pandas as pd

# initialize for the topic on which the msg is received
recv_topic = ""
# initialize a recv msg to store the payload received from MQTT
recv_msg = ""

# define all topics
TOPICS = [
    # end the program
    "topic/Terminate",
    # toggle jog
    "topic/ToggleJog",
    # New Jog
    "topic/Jog",
    #run to
    "topic/runTo",
    #receive change topic signal
    "topic/Break",
    #Emergency Stop
    "topic/CoreFunc",
    #Hold button status for teach point
    "topic/TPMove",
    #Export teach points data
    "topic/recv_data",
    #filename of csv file
    "topic/filename",
]

def main_cb(client, userdata, msg):
    # use `global` to change a variable outside of the callback function
    global recv_topic, recv_msg
    recv_topic = msg.topic
    if recv_topic == "topic/recv_data":
        recv_msg = json.loads(msg.payload)
        print('In callback, msg:', recv_msg)
    else:
        recv_msg = msg.payload.decode('utf-8')
        print('In callback, msg:', recv_msg)

# localhost / 127.0.0.1
host = "localhost"

client = mqtt.Client("MG400")
client.connect(host, port=1883)
client.loop_start()

# Subscribe to all topics
for topic in TOPICS:
    client.subscribe(topic)

client.on_message = main_cb

# def debug_run():
#     def debug_publish():
#         while True:
#             for topic in TOPICS:
#                 client.publish(topic, 'dummy message', qos=1)
#                 time.sleep(0.5)

#                 # the recv_topic & recv_msg can be accessed in main function
#                 # then you can do your stuff with them here
#                 print(f"In main function, topic: {recv_topic}")
#                 print(f"In main function, msg: {recv_msg}\n")

#                 if recv_topic == 'topic/Terminate':
#                     print("TERMINATING")
#                     client.loop_stop()
#                     client.disconnect()
#                     exit()

#     p1 = Thread(target=debug_publish)
#     p1.start()

# Main function where all robot code logic exist
size = 6
Angle = [0] * size
Pose = [0] *size

def main(client_dashboard, client_feedback, Angle, Pose):
    # Remove alarm
    client_dashboard.ClearError()
    time.sleep(0.5)
    # Description The upper function was enabled successfully
    client_dashboard.EnableRobot()
    time.sleep(0.5)

    while True:
        print("Loop 1")
        #Other functions put herse
        Angle = client_dashboard.GetAngle()
        client.publish("topic/Angle", Angle)
        Pose = client_dashboard.GetPose()
        client.publish("topic/Pose",Pose)
        time.sleep(0.25)

        #to unlock robot
        if recv_topic == "topic/CoreFunc" and recv_msg == "Enable":
            client_dashboard.EnableRobot()

        if recv_topic == "topic/CoreFunc" and recv_msg == "Disable":
            client_dashboard.DisableRobot()

        if recv_topic == "topic/CoreFunc" and recv_msg == "ClearErr":
            client_dashboard.ClearError()

        #Teach point
        if recv_topic == "topic/runTo":
            time.sleep(0.25)
            point = recv_msg

           # add additional 2 joint angles as the Dobot API reads 6 joint angles eventhough 4 are used
           # augment the string and change it to an array of joint angles
            pointCompare = "{"+point+",0.000000,0.000000}"
            pointString = point.split(",")

            #topic/TPMove for hold button
            while recv_topic == "topic/runTo" or recv_topic == "topic/TPMove":
                print("Loop 2")
                #Stop robot
                client_feedback.MoveJog("")
                time.sleep(0.20)
                #Re-enable robot
                client_dashboard.EnableRobot()
                time.sleep(0.20)
                Angle = client_dashboard.GetAngle()
                client.publish("topic/Angle", Angle)
                #If button is held down
                while recv_topic == "topic/TPMove" and recv_msg == "GO":
                    print("Loop 3")
                    #trigger the mg400 to move to the designated point
                    client_feedback.JointMovJ2((float(pointString[0])),float((pointString[1])),float((pointString[2])),float((pointString[3])))
                    time.sleep(0.25)
                    Angle = client_dashboard.GetAngle()
                    print(Angle)
                    #if button is released
                    if recv_msg == "NOGO" and recv_topic == "topic/TPMove":
                        break

                    #if receive terminate app
                    if recv_msg == "Terminate":
                        break

                #if reached teach point position
                if Angle == pointCompare:
                    client.publish("topic/break", payload = "Done")
                    time.sleep(0.25)

                #if receive terminate app
                if recv_msg == "Terminate":
                    break
            while recv_topic == "topic/Jog":
                client_feedback.MoveJog(recv_msg)
                time.sleep(0.2)
                Angle = client_dashboard.GetAngle()
                client.publish("topic/Angle", Angle)
                Pose = client_dashboard.GetPose()
                client.publish("topic/Pose",Pose)
                # use sleep to avoid keep sending the MoveJog command to the dobot
                time.sleep(0.2)

                if recv_msg == "Terminate":
                    break
            #Terminate will shut down app
            if recv_msg == "Terminate":
                client_dashboard.DisableRobot()
                client.loop_stop()
                client.disconnect()

                client_dashboard.close()
                client_feedback.close()
                break

            #save filename
            if recv_topic == "topic/filename":
                filename = recv_msg
                filepath = "CSV_SaveLoc"
                fullpath = filepath + filename
            #export data to file
            if recv_topic == "topic/recv_data":
                # access the Dictionary from the parsed JSON
                coord1 = recv_msg[0]['J1']
                coord2 = recv_msg[0]['J2']
                coord3 = recv_msg[0]['J3']
                coord4 = recv_msg[0]['J4']
                labels = recv_msg[0]['labels']
                view = recv_msg[0]['view']

                # save to CSV file
                df = pd.DataFrame([recv_msg]).T
                df.to_csv(fullpath, index=False)

                if recv_topic == "topic/CoreFunc" and recv_msg == "GetData":
                    df = pd.read_csv(fullpath)
                    lines= len(list(df))
                    #df = pd.read_csv('OneDrive/Desktop/Internship/test2.csv')
                    row = 0
                    data_dict = "Random String"
                    while row <= lines:
                        time.sleep(1)
                        data_dict = df.to_dict(orient='records')[row]
                        print(f"{data_dict}")
                        send = str(data_dict)

                        #publish the data in the called csv file to node red
                        client.publish("topic/recv_joint",send)
                        row = row + 1
                    client.publish("topic/break", payload = "Done")

                # #print(pointString)
                # time.sleep(0.25)
                # Angle = client_dashboard.GetAngle()
                # time.sleep(0.25)
                # #client.publish("topic/break", payload = "Break")
                # #print(pointCompare)
                # #print(Angle)
                # time.sleep(0.5)

                # if Angle == pointCompare or (recv_msg == "Stop" and recv_topic == "topic/runTo"):
                #     #client_feedback.MoveJog()
                #     time.sleep(0.25)
                #     client_dashboard.EnableRobot()
                #     client.publish("topic/break", payload = "Done")
                #     time.sleep(0.25)

        # if recv_topic == "topic/ToggleJog":
        #     if recv_msg == "Start":
        #         while True:
        #             if recv_topic == "topic/Jog":
        #                 client_feedback.MoveJog(recv_msg)

        #             time.sleep(0.2)
        #             Angle = client_dashboard.GetAngle()
        #             client.publish("topic/Angle", Angle)
        #             Pose = client_dashboard.GetPose()
        #             client.publish("topic/Pose",Pose)
        #             # use sleep to avoid keep sending the MoveJog command to the dobot
        #             time.sleep(0.2)

        #             if recv_msg == "Terminate":
        #                 break

        #             if recv_topic == "topic/ToggleJog" and recv_msg == "Stop":
        #                 break

# Enable threads on ports 29999 and 30003
if __name__ == '__main__':
    client_dashboard = dobot_api_dashboard('192.168.1.6', 29999)
    client_feedback = dobot_api_feedback('192.168.1.6', 30003)
    p1 = Thread(target=main, args=(client_dashboard, client_feedback, Angle, Pose))
    p1.start()


import datetime
from django.contrib.auth.models import User
from django.core.validators import MaxValueValidator, MinValueValidator
from django.db import models
from django.utils import timezone

class Deal(models.Model):
  CATEGORY_LABEL = (
    ('poc', 'おこずかい'),
    ('sal', '給与'),
    ('oti', 'その他所得'),
    ('foo', '食費'),
    ('boo', '教育'),
    ('tra', '交通費'),
    ('sho', 'ショッピング'),
    ('prs', 'プレゼント'),
    ('ent', '交際費'),
    ('fee', '手数料'),
    ('oto', 'その他消費'),
  )

  # define
  date = models.DateField('日付', default=timezone.now)
  in_out = models.IntegerField('収支', validators=[MinValueValidator(-999999), MaxValueValidator(999999)])
  category = models.CharField(
    'カテゴリー',
    choices=CATEGORY_LABEL,
    default='foo',
    max_length=3,
  )
  description = models.CharField('内容', blank=True, max_length=50)

  user = models.ForeignKey(
    User,
    on_delete=models.SET_NULL,
    null=True,
  )

  def __str__(self):
    return self.description

class Wishlist(models.Model):
  name = models.CharField('名前', max_length=100)
  image = models.ImageField('画像', upload_to='images/', blank=True)
  price = models.IntegerField('価格',
    validators=[
      MinValueValidator(0),
      MaxValueValidator(9999999)
    ],
    blank=True,
  )
  term = models.IntegerField('期間',
    validators=[
      MinValueValidator(1),
      MaxValueValidator(3653)
    ],
    blank=True,
  )
  url = models.CharField('URL', max_length=255, blank=True)
  created_at = models.DateTimeField('作成日', auto_now_add=True)
  memo = models.TextField(
    '備考',
    blank=True,
    max_length=300,
    )

  user = models.ForeignKey(
    User,
    on_delete=models.SET_NULL,
    null=True,
  )


from torch import nn

class Second2Last(object):
    def __call__(self, model: nn.Module):
        l = 0
        layers_dict = {}

        for _ in model.modules():
            l += 1

        for i, layer in enumerate(model.modules()):
            if i == (l - 1):
                layers_dict[layer] = i

        return layers_dict


import os
import sys
import pickle
import numpy as np
import xml.etree.ElementTree as ET
import random
import svgwrite
from IPython.display import SVG, display
import tensorflow as tf

def get_bounds( data, factor ):
  min_x = 0
  max_x = 0
  min_y = 0
  max_y = 0

  abs_x = 0
  abs_y = 0
  for i in range( len( data ) ):
    x = float( data[ i, 0 ] )/factor
    y = float( data[ i, 1 ] )/factor
    abs_x += x
    abs_y += y
    min_x = min( min_x, abs_x )
    min_y = min( min_y, abs_y )
    max_x = max( max_x, abs_x )
    max_y = max( max_y, abs_y )

  return ( min_x, max_x, min_y, max_y )

# version where each path is entire stroke ( smaller svg size, but have to keep same color )
def draw_strokes( data, factor = 10, svg_filename = 'sample.svg' ):
  min_x, max_x, min_y, max_y = get_bounds( data, factor )
  dims = ( 50 + max_x - min_x, 50 + max_y - min_y )

  dwg = svgwrite.Drawing( svg_filename, size = dims )
  dwg.add( dwg.rect( insert = ( 0, 0 ), size = dims, fill = 'white' ) )

  lift_pen = 1

  abs_x = 25 - min_x
  abs_y = 25 - min_y
  p = "M%s, %s " % ( abs_x, abs_y )

  command = "m"

  for i in range( len( data ) ):
    if ( lift_pen == 1 ):
      command = "m"
    elif ( command != "l" ):
      command = "l"
    else:
      command = ""
    x = float( data[ i, 0 ] )/factor
    y = float( data[ i, 1 ] )/factor
    lift_pen = data[ i, 2 ]
    p += command+str( x )+", "+str( y )+" "

  the_color = "black"
  stroke_width = 1

  dwg.add( dwg.path( p ).stroke( the_color, stroke_width ).fill( "none" ) )

  dwg.save( )
  display( SVG( dwg.tostring( ) ) )

def draw_strokes_eos_weighted( stroke, param, factor = 10, svg_filename = 'sample[ A_eos.svg' ):
  c_data_eos = np.zeros( ( len( stroke ), 3 ) )
  for i in range( len( param ) ):
    c_data_eos[ i, : ] = ( 1-param[ i ][ 6 ][ 0 ] )*225 # make color gray scale, darker = more likely to eos
  draw_strokes_custom_color( stroke, factor = factor, svg_filename = svg_filename, color_data = c_data_eos, stroke_width = 3 )

def draw_strokes_random_color( stroke, factor = 10, svg_filename = 'sample_random_color.svg', per_stroke_mode = True ):
  c_data = np.array( np.random.rand( len( stroke ), 3 )*240, dtype = np.uint8 )
  if per_stroke_mode:
    switch_color = False
    for i in range( len( stroke ) ):
      if switch_color == False and i > 0:
        c_data[ i ] = c_data[ i-1 ]
      if stroke[ i, 2 ] < 1: # same strike
        switch_color = False
      else:
        switch_color = True
  draw_strokes_custom_color( stroke, factor = factor, svg_filename = svg_filename, color_data = c_data, stroke_width = 2 )

def draw_strokes_custom_color( data, factor = 10, svg_filename = 'test.svg', color_data = None, stroke_width = 1 ):
  min_x, max_x, min_y, max_y = get_bounds( data, factor )
  dims = ( 50 + max_x - min_x, 50 + max_y - min_y )

  dwg = svgwrite.Drawing( svg_filename, size = dims )
  dwg.add( dwg.rect( insert = ( 0, 0 ), size = dims, fill = 'white' ) )

  lift_pen = 1
  abs_x = 25 - min_x
  abs_y = 25 - min_y

  for i in range( len( data ) ):

    x = float( data[ i, 0 ] )/factor
    y = float( data[ i, 1 ] )/factor

    prev_x = abs_x
    prev_y = abs_y

    abs_x += x
    abs_y += y

    if ( lift_pen == 1 ):
      p = "M "+str( abs_x )+", "+str( abs_y )+" "
    else:
      p = "M +"+str( prev_x )+", "+str( prev_y )+" L "+str( abs_x )+", "+str( abs_y )+" "

    lift_pen = data[ i, 2 ]

    the_color = "black"

    if ( color_data is not None ):
      the_color = "rgb( "+str( int( color_data[ i, 0 ] ) )+", "+str( int( color_data[ i, 1 ] ) )+", "+str( int( color_data[ i, 2 ] ) )+" )"

    dwg.add( dwg.path( p ).stroke( the_color, stroke_width ).fill( the_color ) )
  dwg.save( )
  display( SVG( dwg.tostring( ) ) )

def draw_strokes_pdf( data, param, factor = 10, svg_filename = 'sample_pdf.svg' ):
  min_x, max_x, min_y, max_y = get_bounds( data, factor )
  dims = ( 50 + max_x - min_x, 50 + max_y - min_y )

  dwg = svgwrite.Drawing( svg_filename, size = dims )
  dwg.add( dwg.rect( insert = ( 0, 0 ), size = dims, fill = 'white' ) )

  abs_x = 25 - min_x
  abs_y = 25 - min_y

  num_mixture = len( param[ 0 ][ 0 ] )

  for i in range( len( data ) ):

    x = float( data[ i, 0 ] )/factor
    y = float( data[ i, 1 ] )/factor

    for k in range( num_mixture ):
      pi = param[ i ][ 0 ][ k ]
      if pi > 0.01: # optimisation, ignore pi's less than 1% chance
        mu1 = param[ i ][ 1 ][ k ]
        mu2 = param[ i ][ 2 ][ k ]
        s1 = param[ i ][ 3 ][ k ]
        s2 = param[ i ][ 4 ][ k ]
        sigma = np.sqrt( s1*s2 )
        dwg.add( dwg.circle( center = ( abs_x+mu1*factor, abs_y+mu2*factor ), r = int( sigma*factor ) ).fill( 'red', opacity = pi/( sigma*sigma*factor ) ) )

    prev_x = abs_x
    prev_y = abs_y

    abs_x += x
    abs_y += y

  dwg.save( )
  display( SVG( dwg.tostring( ) ) )

class DataLoader( ):

  def getRandValue( self ):
    value = self.randvalues[ self.randvaluepointer ]
    self.randvaluepointer += 1
    if ( self.randvaluepointer >= self.nrrandvalues ):
        self.randvaluepointer = 0
    return value

  def createRandValues( self ):
    self.nrrandvalues = 1000
    self.randvalues = np.zeros( ( self.nrrandvalues ), dtype = np.float32 )
    for i in range( self.nrrandvalues ):
        value = random.random( )
        self.randvalues[ i ] = value
    self.randvaluepointer = 0

  def getClassLabels( self ):
    if self.train:
      fn = self.data_dir + "trainlabels.txt"
    else:
      fn = self.data_dir + "testlabels.txt"
    classlabels = np.loadtxt( fn )
    classlabels = classlabels[ :self.nrinputfiles ]
    return classlabels

  def findAvailableExamples( self, args ):
    self.availableExamples = [ ]
    findexamples = True
    if findexamples:
      for i in range( len( self.classlabels ) ):
        if ( self.classlabels[ i ] < args.curnrdigits ):
          self.availableExamples.append( i )
    self.availableExamples = np.array( self.availableExamples )

  def __init__( self, datadir, args, totnrfiles, curnrexamples, seqlength = 0, train = 1, file_label = "", print_input = 0, rangemin = 0, rangelen = 0 ):

    random.seed( 100*args.runnr )
    np.random.seed( 100*args.runnr )
    tf.set_random_seed( 100*args.runnr )
    self.args = args

    self.data_dir = datadir
    self.train = train
    if self.train:
      self.traintest = "train"
    else:
      self.traintest = "test"
    self.rangemin = rangemin
    self.rangelen = rangelen
    self.nrinputfiles = totnrfiles

    self.curnrexamples = curnrexamples
    self.nrseq_per_batch = args.nrseq_per_batch
    self.file_label = file_label
    self.print_input = print_input
    self.nrinputvars_data = self.getInputVectorLength( args )
    self.max_seq_length = args.max_seq_length

    self.nrsequenceinputs = 4 #dx dy eos eod
    self.nrauxinputvars = args.nrClassOutputVars #either [ 0..9 dx dy eos eod ] or [ dx dy eos ]

    strokedatafile = os.path.join( self.data_dir, "strokes_"+self.traintest+"ing_data"+ file_label+args.explabel+ ".cpkl" )
    raw_data_dir = self.data_dir+"/lineStrokes"

    print ( "creating data cpkl file from source data" )
    self.preprocess( args, raw_data_dir, strokedatafile )

    if ( seqlength > 0 ): #provided
      self.seq_length = seqlength
    else:
      self.seq_length = min( self.max_seq_length, args.maxdigitlength_nrpoints )

    self.load_preprocessed( args, strokedatafile )

    self.classlabels = self.getClassLabels( )
    self.findAvailableExamples( args )

    self.nrbatches_per_epoch = max( 1, int( self.curnrexamples / self.nrseq_per_batch ) )
    print ( "curnrexamples", self.curnrexamples, "seq_length", self.seq_length, " --> nrbatches_per_epoch: ", self.nrbatches_per_epoch )

    print ( "loaded data" )
    self.reset_batch_pointer( args )

  def constructInputFileName( self, args, file_label, imgnr ):
    filename = self.data_dir + self.traintest + 'img' + file_label + '-' + str( imgnr ) + '-targetdata.txt' #currently, we expect 14 inputs
    return filename

  def getInputVectorLength( self, args ):
    result = [ ]

    filename = self.constructInputFileName( args, self.file_label, imgnr = 0 )

    with open( filename ) as f:
      points = [ ]
      line = f.readline( )
      print ( "read sample line from inputdata file: ", line )
      nrs = [ float( x ) for x in line.split( ) ]
      length = len( nrs )
      print ( "Determined nrinputvars based on data: ", length )
    self.nrinputvars_data = length
    return length

  def preprocess( self, args, data_dir, strokedatafile ):
    filelist = [ ]

    if len( args.fileselection )>0:
      fileselection = ' '.join( args.fileselection )
      if len( fileselection )>0:
        fileselection = [ int( s ) for s in fileselection.split( ', ' ) ]

    for imgnr in range( 0, self.nrinputfiles ):
        if len( args.fileselection )>0:
          fname = self.constructInputFileName( args, self.file_label, fileselection[ imgnr ] )
        else:
          fname = self.constructInputFileName( args, self.file_label, imgnr )
        filelist.append( fname )

    def getStrokes( filename, nrauxinputvars ): #returns array of arrays with points
      result_points = [ ]
      result_auxinputs = [ ]
      nrsequencevars = 4
      dxmin = 1e100
      dxmax = -1e100
      dymin = 1e100
      dymax = -1e100
      nrauxinputs_data = 10

      with open( filename ) as f:
        points = [ ]
        auxinputs = [ ]
        for line in f: # read rest of lines
          nrs = [ float( x ) for x in line.split( ) ]
          auxinputvalues = nrs[ 0:nrauxinputvars ]
          point = nrs[ nrauxinputs_data:nrauxinputs_data+nrsequencevars ] #currently: x, y, end-of-stroke
          points.append( point )
          auxinputs.append( auxinputvalues )
        result_points.append( points )
        result_auxinputs.append( auxinputs )
        pointarray = np.array( points )
        digitlength_nrpoints = len( points )
        dxmin = pointarray[ :, 0 ].min( )
        dxmax = pointarray[ :, 0 ].max( )
        dymin = pointarray[ :, 1 ].min( )
        dymax = pointarray[ :, 1 ].max( )
        ranges = [ dxmin, dxmax, dymin, dymax ]
      return result_auxinputs, result_points, ranges, digitlength_nrpoints

    # converts a list of arrays into a 2d numpy int16 array
    def convert_stroke_to_array( stroke ):

      n_point = 0
      for i in range( len( stroke ) ):
        n_point += len( stroke[ i ] )

      prev_x = 0
      prev_y = 0
      counter = 0
      nrsequencevars = 4
      stroke_data = np.zeros( ( n_point, nrsequencevars ), dtype = np.int16 )

      for j in range( len( stroke ) ):
        for k in range( len( stroke[ j ] ) ):
          for s in range( nrsequencevars ):
            stroke_data[ counter, s ] = int( stroke[ j ][ k ][ s ] )
          counter += 1
      return stroke_data

    # converts a list of arrays into a 2d numpy int16 array
    def convert_auxinputs_to_array( auxinputs, nrauxinputvars ):

      n_point = 0
      for i in range( len( auxinputs ) ):
        n_point += len( auxinputs[ i ] )
      auxinputdata = np.zeros( ( n_point, nrauxinputvars ), dtype = np.int16 )

      prev_x = 0
      prev_y = 0
      counter = 0

      for j in range( len( auxinputs ) ):
        for k in range( len( auxinputs[ j ] ) ):
          for a in range( nrauxinputvars ):
            auxinputdata[ counter, a ] = int( auxinputs[ j ][ k ][ a ] )
          counter += 1
      return auxinputdata

    # preprocess body: build stroke array
    strokearray = [ ]
    auxinputarray = [ ]
    rangelist = [ ]
    self.seqlengthlist = [ ]
    if self.train:
      args.maxdigitlength_nrpoints = 0
    digitlengthsum = 0
    for i in range( len( filelist ) ):
      print ( 'dataloader', self.traintest, 'processing '+filelist[ i ] )
      [ auxinputs, strokeinputs, ranges, digitlength_nrpoints ] = getStrokes( filelist[ i ], self.nrauxinputvars )
      strokearray.append( convert_stroke_to_array( strokeinputs ) )
      auxinputarray.append( convert_auxinputs_to_array( auxinputs, self.nrauxinputvars ) )
      rangelist.append( ranges )
      self.seqlengthlist.append( digitlength_nrpoints )
      if self.train:
        args.maxdigitlength_nrpoints = max( args.maxdigitlength_nrpoints, digitlength_nrpoints )
        digitlengthsum += digitlength_nrpoints

    rangearray = np.array( rangelist )
    ranges = [ rangearray[ :, 0 ].min( ), rangearray[ :, 1 ].max( ), rangearray[ :, 2 ].min( ), rangearray[ :, 3 ].max( ) ]
    print ( "found overall ranges", ranges )
    self.avgseqlength = digitlengthsum / len( filelist )
    print( "dataloader: found avg seq length: ", self.avgseqlength )
    print ( "found maxdigitlength_nrpoints", args.maxdigitlength_nrpoints )

    f = open( strokedatafile, "wb" )
    pickle.dump( strokearray, f )
    pickle.dump( auxinputarray, f )
    pickle.dump( ranges, f )
    pickle.dump( self.seqlengthlist, f )
    f.close( )

  def load_preprocessed( self, args, strokedatafile ):
    f = open( strokedatafile, "rb" )
    self.strokedataraw = pickle.load( f )
    self.auxdataraw = pickle.load( f )
    self.ranges = pickle.load( f )
    self.seqlengthlist = pickle.load( f )
    f.close( )

    print ( "loaded ranges", self.ranges )
    print ( "rangemin", self.rangemin, "rangelen", self.rangelen )

    self.strokedata = [ ] #contains one array per file
    self.auxdata = [ ]
    counter = 0

    for data_el in self.strokedataraw:
        data = np.array( np.zeros( ( self.seq_length, self.nrsequenceinputs ), dtype = np.float32 ) )
        len_data = len( data )
        nrpoints = min( self.seq_length, len( data_el ) )
        data[ :nrpoints, ] = data_el[ :nrpoints ]
        if ( len( data_el ) > self.seq_length ) and ( self.seq_length >= args.max_seq_length ):
          data[ self.seq_length-1, 2:4 ] = np.ones( ( 1, 2 ), dtype = np.float32 ) #add eos and eod for sequences exceeding length
        data[ nrpoints:, 0:4 ] = np.zeros( ( len_data - nrpoints, 4 ), dtype = np.float32 ) #pad remainder with zero rows
        data[ :, 0:2 ] -= self.rangemin
        data[ :, 0:2 ] /= self.rangelen
        self.strokedata.append( data )

        counter += 1
    for data_el in self.auxdataraw:
        data = np.array( np.zeros( ( self.seq_length, self.nrauxinputvars ), dtype = np.float32 ) )
        nrpoints = min( self.seq_length, len( data_el ) )
        data[ :nrpoints, ] = data_el[ :nrpoints ]
        data[ nrpoints:self.seq_length, ] = data[ nrpoints-1, ]
        self.auxdata.append( data )
    print ( "#sequences found in data: ", counter )

  def next_batch( self, args, curseqlength ):

    # returns a batch of the training data of nrseq_per_batch * seq_length points
    x_batch = [ ]
    y_batch = [ ]
    seqlen = self.seq_length
    sequence_index = [ ]
    use_points_stopcrit = False

    nrpoints_per_batch = 0
    if hasattr( args, 'nrpoints_per_batch' ):
      nrpoints_per_batch = args.nrpoints_per_batch
    if nrpoints_per_batch > 0:
      use_points_stopcrit = True
    batch_nrpoints = 0
    batch_sequencenr = 0
    done = False
    while not done:
      sequence_index.append( self.pointer )
      strokes = np.copy( self.strokedata[ self.pointer ] )
      auxvalues = np.copy( self.auxdata[ self.pointer ] )

      if args.useStrokeOutputVars:
        ytab = np.copy( np.hstack( [ auxvalues[ 1:seqlen ], strokes[ 1:seqlen ] ] ) )
      else:
        ytab = np.copy( np.hstack( [ auxvalues[ 1:seqlen ] ] ) )

      if args.discard_classvar_inputs:
        auxvalues[ : ] = 0

      if args.useClassInputVars:
        xtab = np.hstack( [ auxvalues[ :seqlen-1 ], strokes[ :seqlen-1 ] ] )
      else:
        xtab = strokes[ :seqlen-1 ]

      actual_seq_length = self.seqlengthlist[ self.pointer ]

      firsttrainstep = 0
      if hasattr( args, 'firsttrainstep' ):
        firsttrainstep = args.firsttrainstep
      firsttrainstep = min ( firsttrainstep, actual_seq_length - 1 )
      if firsttrainstep > 0: #remove earlier part from _target_ data, so that it will not be used in loss.
        ytab[ :firsttrainstep, : ] = 0

      #only keep points up to current seq_length - 1; e.g. if sequence has 3 points, use 2 pairs of ( x, y ): k = 1.n-1 for x and k = 2..n for y
      firstafter = min( actual_seq_length - 1, curseqlength ) #zero out part after sequence

      xtab[ firstafter:, : ] = 0
      ytab[ firstafter:, : ] = 0

      nrusedpoints = firstafter - firsttrainstep

      if args.discard_inputs:
        xtab[ : ] = 0

      x_batch.append( np.copy( xtab ) )
      y_batch.append( np.copy( ytab ) )

      self.next_batch_pointer( args )

      batch_sequencenr += 1
      nrseq_per_batch = self.nrseq_per_batch
      if ( not self.train ) and hasattr( args, 'nrseq_per_batch_test' ):
        nrseq_per_batch = args.nrseq_per_batch_test

      if use_points_stopcrit:
        batch_nrpoints += nrusedpoints
        done = batch_nrpoints >= nrpoints_per_batch
      else:
        done = batch_sequencenr >= nrseq_per_batch

    return x_batch, y_batch, sequence_index

  def selectExamples( self, nrdigits ):
    sample = np.random.permutation( len( self.availableExamples ) )
    return self.availableExamples[ sample ]

  def next_batch_pointer( self, args ):
    self.index += 1
    if ( self.index >= len( self.example_permutation ) ):
      self.reset_batch_pointer( args )
    self.pointer = self.example_permutation[ self.index ]

  def reset_batch_pointer( self, args ):
    self.index = 0

    if ( args.incremental_nr_digits and self.train ):
      self.example_permutation = self.selectExamples( args.curnrdigits )
    else:
      if self.train:
        self.example_permutation = np.random.permutation( int( self.curnrexamples ) )
      else:
        self.example_permutation = np.arange( 0, int( self.curnrexamples ) )
    self.pointer = self.example_permutation[ self.index ]


from io import StringIO
import unittest
from unittest.mock import patch

from src.utils.cli import ask_yes_or_no

class TestCli(unittest.TestCase):
    @patch('sys.stdout', new_callable=StringIO)
    def test_valid_message_print(self, mock_stdout):
        with patch('builtins.input', return_value='yes'):
            respuesta = ask_yes_or_no('Continue? (yes/no): ')
        self.assertEqual(respuesta, True)
        self.assertEqual(mock_stdout.getvalue(), 'Continue? (yes/no): \n')

    def test_valid_response_yes(self):
        with patch('builtins.input', return_value='yes'):
            respuesta = ask_yes_or_no('Continue? (yes/no): ')
        self.assertEqual(respuesta, True)

    def test_valid_response_no(self):
        with patch('builtins.input', return_value='no'):
            respuesta = ask_yes_or_no('Continue? (yes/no): ')
        self.assertEqual(respuesta, False)

    @patch('sys.stdout', new_callable=StringIO)
    def test_invalid_response(self, mock_stdout):
        with patch('builtins.input', side_effect=['something else', 'yes']):
            respuesta = ask_yes_or_no('a ')
        self.assertEqual(respuesta, True)
        self.assertEqual(mock_stdout.getvalue(), 'a \nPlease enter \'yes\' or \'no\'\na \n')


from asyncio import Queue

async def async_generator_from_queue(queue: Queue, stop_value=None):
    while True:
        value = await queue.get()

        if value is stop_value:
            return
        else:
            yield value
            queue.task_done()


# -*- coding: utf-8 -*-

import sys
import os
import shlex
import re
import subprocess

from os.path import abspath, isfile

# http://docs.readthedocs.io/en/latest/getting_started.html#in-markdown
from recommonmark.parser import CommonMarkParser
source_parsers = { '.md': CommonMarkParser, }

sys.path.insert(0, abspath('.'))
sys.path.insert(0, abspath('_extensions'))

# -- General configuration ------------------------------------------------

needs_sphinx = '1.5'

extensions = [
    # Standard Sphinx extensions
    'sphinx.ext.extlinks',
    'sphinx.ext.intersphinx',
    'sphinx.ext.todo',
    'sphinx.ext.graphviz',
    'sphinx.ext.mathjax',
    'sphinx.ext.ifconfig',
    'sphinx.ext.viewcode',
    # 'sphinx.ext.githubpages',
    # SphinxContrib extensions
    # 'sphinxcontrib.textstyle',
    # 'sphinxcontrib.spelling',
]

source_suffix = ['.rst', '.md']

master_doc = 'index'

project = u'BTD'
copyright = u'2018, Build The Docs'
author = u''

version = "0.0"    # The short X.Y version.
release = version  # The full version, including alpha/beta/rc tags.

language = None

# There are two options for replacing |today|: either, you set today to some non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

exclude_patterns = []

pygments_style = 'stata-dark'

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = True
todo_link_only = True

# reST settings
prologPath = "prolog.inc"
try:
    with open(prologPath, "r") as prologFile:
        rst_prolog = prologFile.read()
except Exception as ex:
    print("[ERROR:] While reading '{0!s}'.".format(prologPath))
    print(ex)
    rst_prolog = ""

# -- Options for HTML output ----------------------------------------------

html_theme_path = ["."]
html_theme = "btd.sphinx.theme"

html_context = {
    # Override default css
    # 'css_files': [
    #     '_static/theme_overrides.css',
    # ]
}

html_context = {
    'use_gfonts': True,
    'display_github': True,
    'slug_user': 'buildthedocs',
    'slug_repo': 'btd',
    'slug_path': 'master/doc/',
    'description': 'Build the Docs',
    'current_version': 'master'
}

# Add display_github and VERSIONING data
if isfile('context.json'):
    from json import loads
    data = loads(open('context.json').read())
    html_context.update(data)

html_theme_options = {
    'style_nav_header_background': '#e7e7e7ff',
    'logo_only': True,
    'home_breadcrumbs': False,
    # 'home_logo': False,
    'prevnext_location': 'bottom'
}

# html_title = None # Defaults to "<project> v<release> documentation".
# html_short_title = None
# html_logo = None
# html_favicon = None

html_static_path = ['_static']

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
html_last_updated_fmt = '%b %d, %Y'

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# Output file base name for HTML help builder.
htmlhelp_basename = 'BTDdoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
    'papersize': 'a4paper',
    'pointsize': '10pt',
    # 'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto, manual, or own class]).
latex_documents = [
  ('index', 'BTD.tex', u'BTD Documentation', [author], 'manual'),
]

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, 'btd', u'BTD Documentation', [author], 1)
]

# If true, show URL addresses after external links.
# man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author, dir menu entry, description, category)
texinfo_documents = [
  (master_doc, 'BTD', u'BTD Documentation', author, 'BTD', 'Build The Docs.', 'Miscellaneous'),
]

# -- Sphinx.Ext.ExtLinks --------------------------------------------------

extlinks = {
   'wikipedia': ('https://en.wikipedia.org/wiki/%s', None),
   'btdsharp': ('https://github.com/buildthedocs/btd/issues/%s', '#'),
   'btdissue': ('https://github.com/buildthedocs/btd/issues/%s', 'issue #'),
   'btdpull':  ('https://github.com/buildthedocs/btd/pull/%s', 'pull request #'),
   'btdrepo':   ('https://github.com/buildthedocs/btd/blob/master/%s', None)
}