#!/usr/bin/python3

from daemons import Daemon, Wallet

import random
import time
import shutil
import os
from os import path
import asyncio
import glob
from datetime import datetime
import uuid

datadirectory="testdata"

def coins(*args):
    if len(args) != 1:
        return tuple(coins(x) for x in args)
    x = args[0]
    if type(x) in (tuple, list):
        return type(x)(coins(i) for i in x)
    return round(x * 1000000000)

def wait_for(callback, timeout=10):
    expires = time.time() + timeout
    while True:
        try:
            if callback():
                return
        except:
            pass
        if time.time() >= expires:
            raise RuntimeError("task timeout expired")
        time.sleep(.25)

verbose = True
def vprint(*args, timestamp=True, **kwargs):
    global verbose
    if verbose:
        if timestamp:
            print(datetime.now(), end=" ")
        print(*args, **kwargs)

class MNNetwork:
    def __init__(self, datadir, *, binpath='../../build/bin', mns=12, nodes=3):
        self.datadir = datadir
        if not os.path.exists(self.datadir):
            os.makedirs(self.datadir)
        self.binpath = binpath

        vprint("Using '{}' for data files and logs".format(datadir))

        nodeopts = dict(master=self.binpath+'/master', datadir=datadir)

        self.mns = [Daemon(master_node=True, **nodeopts) for _ in range(mns)]
        self.nodes = [Daemon(**nodeopts) for _ in range(nodes)]

        self.all_nodes = self.mns + self.nodes

        self.wallets = []
        for name in ('Alice', 'Bob', 'Mike'):
            self.wallets.append(Wallet(
                node=self.nodes[len(self.wallets) % len(self.nodes)],
                name=name,
                rpc_wallet=self.binpath+'/beldex-wallet-rpc',
                datadir=datadir))

        self.alice, self.bob, self.mike = self.wallets

        # Interconnections
        for i in range(len(self.all_nodes)):
            for j in (2, 3, 5, 7, 11):
                k = (i + j) % len(self.all_nodes)
                if i != k:
                    self.all_nodes[i].add_peer(self.all_nodes[k])

        vprint("Starting new master master nodes with RPC on {} ports".format(self.mns[0].listen_ip), end="")
        for mn in self.mns:
            vprint(" {}".format(mn.rpc_port), end="", flush=True, timestamp=False)
            mn.start()
        vprint(timestamp=False)
        vprint("Starting new regular master nodes with RPC on {} ports".format(self.nodes[0].listen_ip), end="")
        for d in self.nodes:
            vprint(" {}".format(d.rpc_port), end="", flush=True, timestamp=False)
            d.start()
        vprint(timestamp=False)

        vprint("Waiting for all master's to get ready")
        for d in self.all_nodes:
            d.wait_for_json_rpc("get_info")

        vprint("Beldexds are ready. Starting wallets")

        for w in self.wallets:
            vprint("Starting new RPC wallet {w.name} at {w.listen_ip}:{w.rpc_port}".format(w=w))
            w.start()
        for w in self.wallets:
            w.ready()
            w.refresh()
            vprint("Wallet {w.name} is ready: {a}".format(w=w, a=w.address()))

        for w in self.wallets:
            w.wait_for_json_rpc("refresh")

        # Mine some blocks; we need 100 per MN registration, and we can nearly 600 on fakenet before
        # it hits HF16 and kills mining rewards.  This lets us submit the first 5 MN registrations a
        # MN (at height 40, which is the earliest we can submit them without getting an occasional
        # spurious "Not enough outputs to use" error).
        # to unlock and the rest to have enough unlocked outputs for mixins), then more some more to
        # earn MN rewards.  We need 100 per MN registration, and each mined block gives us an input
        # of 18.9, which means each registration requires 6 inputs.  Thus we need a bare minimum of
        # 6(N-5) blocks, plus the 30 lock time on coinbase TXes = 6N more blocks (after the initial
        # 5 registrations).
        self.mine(100)
        vprint("Submitting first round of master node registrations: ", end="", flush=True)
        for mn in self.mns[0:5]:
            self.mike.register_mn(mn)
            vprint(".", end="", flush=True, timestamp=False)
        vprint(timestamp=False)
        if len(self.mns) > 5:
            vprint("Going back to mining", flush=True)

            self.mine(6*len(self.mns))

            self.print_wallet_balances()

            vprint("Submitting more master node registrations: ", end="", flush=True)
            for mn in self.mns[5:-1]:
                self.mike.register_mn(mn)
                vprint(".", end="", flush=True, timestamp=False)
            vprint(timestamp=False)
            vprint("Done.")

        self.print_wallet_balances()

        vprint("Mining 40 blocks (registrations + flash quorum lag) and waiting for nodes to sync")
        self.sync_nodes(self.mine(40))

        self.print_wallet_balances()

        vprint("Sending fake belnet/ss pings")
        for mn in self.mns:
            mn.ping()

        all_master_nodes_proofed = lambda mn: all(x['quorumnet_port'] > 0 for x in
                mn.json_rpc("get_n_master_nodes", {"fields":{"quorumnet_port":True}}).json()['result']['master_node_states'])

        vprint("Waiting for proofs to propagate: ", end="", flush=True)
        for mn in self.mns:
            wait_for(lambda: all_master_nodes_proofed(mn), timeout=120)
            vprint(".", end="", flush=True, timestamp=False)
        vprint(timestamp=False)
        for mn in self.mns[-1:]:
            self.mike.register_mn(mn)
            vprint(".", end="", flush=True, timestamp=False)
        self.sync_nodes(self.mine(1))
        time.sleep(10)
        for mn in self.mns:
            mn.send_uptime_proof()
        vprint("Done.")

        vprint("Local Devnet MN network setup complete!")
        vprint("Communicate with daemon on ip: {} port: {}".format(self.mns[0].listen_ip,self.mns[0].rpc_port))
        configfile=self.datadir+'config.py'
        with open(configfile, 'w') as filetowrite:
            filetowrite.write('#!/usr/bin/python3\n# -*- coding: utf-8 -*-\nlisten_ip=\"{}\"\nlisten_port=\"{}\"\nwallet_listen_ip=\"{}\"\nwallet_listen_port=\"{}\"\nwallet_address=\"{}\"\nexternal_address=\"{}\"'.format(self.mns[0].listen_ip,self.mns[0].rpc_port,self.mike.listen_ip,self.mike.rpc_port,self.mike.address(),self.bob.address()))

    def refresh_wallets(self, *, extra=[]):
        vprint("Refreshing wallets")
        for w in self.wallets + extra:
            w.refresh()
        vprint("All wallets refreshed")

    def mine(self, blocks=None, wallet=None, *, sync=False):
        """Mine some blocks to the given wallet (or self.mike if None) on the wallet's daemon.
        Returns the daemon's height after mining the blocks.  If blocks is omitted, mines enough to
        confirm regular transfers (i.e. 10 blocks).  If sync is specified, sync all nodes and then
        refresh all wallets after mining."""
        if wallet is None:
            wallet = self.mike
        if blocks is None:
            blocks = 10
        node = wallet.node
        vprint("Mining {} blocks to wallet {.name}".format(blocks, wallet))
        start_height = node.height()
        end_height = start_height + blocks
        node.mine_blocks(blocks, wallet)
        while node.rpc("/mining_status").json()["active"]:
            height = node.height()
            vprint("Mined {}/{}".format(height, end_height))
            time.sleep(0.05 if height >= end_height else 0.25)
        height = node.height()
        vprint("Mined {}/{}".format(height, end_height))

        if sync:
            self.sync_nodes(height)
            self.refresh_wallets()

        return height

    def sync_nodes(self, height=None, *, extra=[], timeout=10):
        """Waits for all nodes to reach the given height, typically invoked after mine()"""
        nodes = self.all_nodes + extra
        heights = [x.height() for x in nodes]
        if height is None:
            height = max(heights)
        if min(heights) >= height:
            vprint("All nodes already synced to height >= {}".format(height))
            return
        vprint("Waiting for all nodes to sync to height {}".format(height))
        last = None
        expiry = time.time() + timeout
        while nodes and time.time() < expiry:
            if heights[-1] < height:
                heights[-1] = nodes[-1].height()
            if heights[-1] >= height:
                heights.pop()
                nodes.pop()
                last = None
                continue
            if heights[-1] != last:
                vprint("waiting for {} [{} -> {}]".format(nodes[-1].name, heights[-1], height))
                last = heights[-1]
            time.sleep(0.1)
        if nodes:
            raise RuntimeError("Timed out waiting for node syncing")
        vprint("All nodes synced to height {}".format(height))

    def sync(self, extra_nodes=[], extra_wallets=[]):
        """Synchronizes everything: waits for all nodes to sync, then refreshes all wallets.  Can be
        given external wallets/nodes to sync."""
        self.sync_nodes(extra=extra_nodes)
        self.refresh_wallets(extra=extra_wallets)

    def print_wallet_balances(self):
        """Instructs the wallets to refresh and prints their balances (does nothing in non-verbose mode)"""
        global verbose
        if not verbose:
            return
        vprint("Balances:")
        for w in self.wallets:
            b = w.balances(refresh=True)
            vprint("    {:5s}: {:.9f} (total) with {:.9f} (unlocked)".format(
                w.name, b[0] * 1e-9, b[1] * 1e-9))

    def __del__(self):
        for n in self.all_nodes:
            n.terminate()
        for w in self.wallets:
            w.terminate()

mnn = None

def run():
    global mnn, verbose
    if not mnn:
        if path.isdir(datadirectory+'/'):
            shutil.rmtree(datadirectory+'/', ignore_errors=False, onerror=None)
        vprint("new MNN")
        mnn = MNNetwork(datadir=datadirectory+'/')
    else:
        vprint("reusing MNN")
        mnn.alice.new_wallet()
        mnn.bob.new_wallet()

        # Flush pools because some tests leave behind impossible txes
        for n in mnn.all_nodes:
            assert n.json_rpc("flush_txpool").json()['result']['status'] == 'OK'

        # Mine a few to clear out anything in the mempool that can be cleared
        mnn.mine(5, sync=True)

        vprint("Alice has new wallet: {}".format(mnn.alice.address()))
        vprint("Bob   has new wallet: {}".format(mnn.bob.address()))

    input("Use Ctrl-C to exit...")
    loop = asyncio.get_event_loop()
    try:
        loop.run_forever()
    except KeyboardInterrupt:
        print(f'!!! AsyncApplication.run: got KeyboardInterrupt during start')
    finally:
        loop.close()

# Shortcuts for accessing the named wallets
def alice(net):
    return net.alice

def bob(net):
    return net.bob

def mike(net):
    return net.mike

if __name__ == '__main__':
    run()


# -*- coding: utf-8 -*-
"""
Created on Mon Mar 27 18:51:59 2017

@author: Johann Dolivet

Natural Language Processing : Sentiment analysis

The goal is to predict if a critic for a movie is positive or negative.

Datas :
    25000 critics (text with label 1 if positive and 0 if negative)
        are used to train the model.
    25000 critics (text without labels) are used to test the model.

Model :
    The Stochastic Gradient Descent Classifier is used.

Libraries : pandas to use dataframes, read csv, transform in csv
            sklearn to transfrom in vectors and to use SGD Classifier
"""
import os
import re
import pandas as pd
from sklearn.linear_model import SGDClassifier
from sklearn.feature_extraction.text import TfidfVectorizer

# paths for the datas
train_path = "public/aclImdb/train/"    # files are under this directory
test_path = "public/imdb_te.csv"        # test data for grade evaluation

# parameter for the fitting :
MinDf = 2       # discard words appearing in less than MinDf documents
MaxDf = 0.9     # discard words appearing in more than MaxDf of the documents

#==============================================================================
# Process the datas
#   convert the datas in a csv (in lowercase without stopwords)
#==============================================================================

def stop_words(fileName):
    result = []
    fichierManip = open(fileName, 'r')
    for line in fichierManip:
        result.append(line[:-1])
    fichierManip.close()
    return result

def imdb_data_preprocess(inpath, name="imdb_tr.csv"):
    chemin = inpath
    stopList = stop_words("stopwords.en.txt")
    negatif = os.path.join(chemin,"neg")
    positif = os.path.join(chemin,"pos")
    signe = [positif, negatif]
    pattern = re.compile('[^a-z]')
    listText, listPol = [],[]
    for elt in signe:
        listeFile = os.listdir(elt)
        for file in listeFile:
            sample = open(elt+'/'+file,'r')
            for line in sample:
                line = line.replace('

', ' ').lower()
                liste = pattern.split(line)
                liste = [elt for elt in liste if (elt not in stopList and len(elt) > 1)]
                line = ' '.join(liste)
                listText.append(line)
            if elt == positif:
                listPol.append(1)
            else:
                listPol.append(0)
            sample.close()
    df = pd.DataFrame({'text' : listText, 'polarity' : listPol})
    df = df[['text','polarity']]
    df.to_csv(name)
#imdb_data_preprocess(train_path)

def imdb_data_preprocessTest(inpath, name="imdb_te.csv"):
    stopList = stop_words("stopwords.en.txt")
    datas = pd.read_csv(inpath, encoding='latin1')
    listCritiques = []
    pattern = re.compile('[^a-z]')
    for idx, row in datas.iterrows():
        texte = row[1].replace('

', ' ').lower()
        liste = pattern.split(texte)
        liste = [elt for elt in liste if (elt not in stopList and len(elt) > 1)]
        line = ' '.join(liste)
        listCritiques.append(line)
    df = pd.DataFrame({'text' : listCritiques})
    df.to_csv(name)
#imdb_data_preprocessTest(test_path)

#==============================================================================
# Unigram method
#   input : dataframes for train and test
#       training :  vector for X (n samples of list of words)
#                   vector for y (labels)
#       testing :   vector for X (n samples of list of words)
#   write the result (prediction for testing dataframe) in a csv file
#==============================================================================

def unigram(dataTrain, dataTest):
    vectorizer = TfidfVectorizer(min_df=MinDf, max_df = MaxDf)
    train = dataTrain['text']
    test = dataTest['text']
    label = dataTrain['polarity']
    trainX = vectorizer.fit_transform(train)
    testX = vectorizer.transform(test)
    clf = SGDClassifier(loss="hinge", penalty="l1")
    clf.fit(trainX, label)
    result = clf.predict(testX)
    fichier = open("unigram.output.txt", 'w')
    for elt in result:
        fichier.write(str(elt) + '\n')
    fichier.close()
#dataTrain = pd.read_csv(trainFile, encoding='latin1')
#dataTest = pd.read_csv(testFile, encoding='latin1')
#unigram(dataTrain, dataTest)

#==============================================================================
# Bigram method
#   input : dataframes for train and test
#       training :  vector for X (n samples of list of words)
#                   vector for y (labels)
#       testing :   vector for X (n samples of list of words)
#   write the result (prediction for testing dataframe) in a csv file
#==============================================================================

def bigram(dataTrain, dataTest):
    vectorizer = TfidfVectorizer(min_df=MinDf, max_df = MaxDf, ngram_range=(1,2))
    train = dataTrain['text']
    test = dataTest['text']
    label = dataTrain['polarity']
    trainX = vectorizer.fit_transform(train)
    testX = vectorizer.transform(test)
    clf = SGDClassifier(loss="hinge", penalty="l1")
    clf.fit(trainX, label)
    result = clf.predict(testX)
    fichier = open("bigram.output.txt", 'w')
    for elt in result:
        fichier.write(str(elt) + '\n')
    fichier.close()
#dataTrain = pd.read_csv(trainFile, encoding='latin1')
#dataTest = pd.read_csv(testFile, encoding='latin1')
#bigram(dataTrain, dataTest)

#==============================================================================
# Unigram with tf-idf method
#   input : dataframes for train and test
#       training :  vector for X (n samples of list of words)
#                   vector for y (labels)
#       testing :   vector for X (n samples of list of words)
#   write the result (prediction for testing dataframe) in a csv file
#==============================================================================

def unigramtfidf(dataTrain, dataTest):
    vectorizer = TfidfVectorizer(min_df=MinDf, max_df = MaxDf, sublinear_tf=True, use_idf=True)
    train = dataTrain['text']
    test = dataTest['text']
    label = dataTrain['polarity']
    trainX = vectorizer.fit_transform(train)
    testX = vectorizer.transform(test)
    clf = SGDClassifier(loss="hinge", penalty="l1")
    clf.fit(trainX, label)
    result = clf.predict(testX)
    fichier = open("unigramtfidf.output.txt", 'w')
    for elt in result:
        fichier.write(str(elt) + '\n')
    fichier.close()
#dataTrain = pd.read_csv(trainFile, encoding='latin1')
#dataTest = pd.read_csv(testFile, encoding='latin1')
#unigramtfidf(dataTrain, dataTest)

#==============================================================================
# Bigram with tf-idf method
#   input : dataframes for train and test
#       training :  vector for X (n samples of list of words)
#                   vector for y (labels)
#       testing :   vector for X (n samples of list of words)
#   write the result (prediction for testing dataframe) in a csv file
#==============================================================================

def bigramtfidf(dataTrain, dataTest):
    vectorizer = TfidfVectorizer(min_df=MinDf, max_df = MaxDf, ngram_range=(1,2), sublinear_tf=True, use_idf=True)
    train = dataTrain['text']
    test = dataTest['text']
    label = dataTrain['polarity']
    trainX = vectorizer.fit_transform(train)
    testX = vectorizer.transform(test)
    clf = SGDClassifier(loss="hinge", penalty="l1")
    clf.fit(trainX, label)
    result = clf.predict(testX)
    fichier = open("bigramtfidf.output.txt", 'w')
    for elt in result:
        fichier.write(str(elt) + '\n')
    fichier.close()
#dataTrain = pd.read_csv(trainFile, encoding='latin1')
#dataTest = pd.read_csv(testFile, encoding='latin1')
#bigramtfidf(dataTrain, dataTest)

#==============================================================================
# The main function to compile everything
#==============================================================================

def main():
#    # construct the imdb_tr.csv
#    imdb_data_preprocess(train_path)
#    imdb_data_preprocessTest(test_path)
    dataTrain = pd.read_csv('imdb_tr.csv', encoding='latin1')
    dataTest = pd.read_csv('imdb_te.csv', encoding='latin1')
    unigram(dataTrain, dataTest)
    bigram(dataTrain, dataTest)
    unigramtfidf(dataTrain, dataTest)
    bigramtfidf(dataTrain, dataTest)

if (__name__ == '__main__'):
    main()

Python/Artificial Intelligence/Project 5 - Natural Language Processing/driver_3.py


import time

import numpy as np
import matplotlib.pyplot as plt
import matplotlib.ticker as ticker
from mpl_toolkits.mplot3d import Axes3D

import pathlib
import os

import jetFunctions as pp

# Enter variables and directory of files
L = 0.03
low = 0
high = 68

dir = 'C:/Users/yurko/Desktop/Repositories/get/10-jet/'

# Soft files by last change
files = os.listdir(dir + 'DATA/')

for j in range(len(files)):
    for i in range (len(files)-1):
        if os.stat(dir + 'DATA/'+ files[i+1]).st_mtime < os.stat(dir + 'DATA/' + files[i]).st_mtime:
            a = files[i+1]
            files[i+1] = files[i]
            files[i] = a
print(files)

# Load data from files
HP = np.loadtxt(dir + 'DATA/' + files[8])
LP = np.loadtxt(dir + 'DATA/' + files[9])

files = files[:8]
data = []

for i in range (len(files)):
    data.append(np.loadtxt(dir + 'DATA/' + files[i]))

# Calculate mean and k
meanLP = sum(LP)/len(LP)
meanHP = sum(HP)/len(HP)

k = (high - low)/(meanHP - meanLP)

# Smoothing plots
dataSP = []
N1 = 20

for i in range (len(data)):
    dataSP.append((np.convolve(data[i], np.ones((N1,))/N1, mode = 'valid') - 78))

data = data - 240/k # Смещение на ноль по оси Oy

# Centering and creating lengthlines
lengths = []
num = len(data[0])-N1+1 # num of elements in smoothed plots
num = len(data[0]) # num of elements in norm plots

# Смещение на ноль по оси Ox
for i in range (len(data)):
    lengths.append( np.linspace(-L/2, L/2, num) - list(dataSP[i]).index(np.max(dataSP[i]))*L/(num) + 0.0125 )

lengths[0] = (np.linspace(-L/2, L/2, num) - list(dataSP[0]).index(np.max(dataSP[0]))*L/(num) + 0.011)
lengths[1] = (np.linspace(-L/2, L/2, num) - list(dataSP[1]).index(np.max(dataSP[1]))*L/(num) + 0.011)

# Calculate jet flow
Q = [0]*8

for j in range (len(data)):  # 8
    for i in range(len(data[j])): # 100

        Q[j] += abs( (L/len(data[j])) * (abs(lengths[j][i])) * (abs((k*data[j][i]-low)*2/1.27))**(1/2) )

Q = [i*1.27*np.pi for i in Q]

# Create 2D plot
fig = plt.figure()
ax = fig.add_subplot(111)
ax.grid(color = 'gray', linestyle = ':')
ax.set(title = 'Центрированный график зависимости P(x)', xlabel = 'x, м', ylabel = 'Давление P, Па')

for i in range(len(dataSP)):
    ax.plot(lengths[i], k*data[i], label = '{}mm'.format(10*i+1))

ax.legend()

# Create 3D plot
fig3D = plt.figure()
ax3D = fig3D.add_subplot(111, projection='3d')
ax3D.set(title = 'Распределение скоростей потока в затопленной струе', xlabel = 'x, м', ylabel = 'Расстояние от сопла l, м', zlabel = 'Скорость потока V, м/с')

for i in range(len(data)):
    ax3D.plot(lengths[i], [(10*i+1)*0.001]*(num), abs(((k*data[i]-low)*2/1.27))**(1/2))

# Create plot of jet flow
figJet = plt.figure()
axJet = figJet.add_subplot(111)
axJet.set(title = 'График зависимости Q(l)', xlabel = 'Расстояние от сопла l, м', ylabel = 'Расход Q, кг/с')
axJet.grid(color = 'gray', linestyle = ':')

axJet.plot(np.linspace(0.001, 0.071, 8), Q)

plt.show()

#Save plots
fig.savefig(dir + '10-jet-Plots/all.png')
fig3D.savefig(dir + '10-jet-Plots/3Dall.png')
figJet.savefig(dir + '10-jet-Plots/Jet.png')


#!/usr/bin/python

import logging, os

# Raspberry Pi SPI Port and Device
spi_port = 0
spi_dev = 0

# Pin # for relay connected to heating element
he_pin = 26

# Default goal temperature
#set_temp = 221.
set_temp = 203.
set_temp_tolerance = 2.
cold_state = 100
warm_state = 200

pid_history_length = 10
temp_history_length = 5

startup_sleep_time = 1

# Default sleep/wake times
sched_enabled = True
sched_disabled_op = "wakeup"
#sched_disabled_op = "gotosleep"
weekday_wake_time = '07:30'
weekday_sleep_time = '10:30'
weekend_wake_time = '10:00'
weekend_sleep_time = '12:30'
schedule_sample_time = 1

# Main loop sample rate in seconds
sample_time = 0.5
# Number of seconds until machine changes between hot & cold states.
num_secs_for_state_change = 60*15
num_secs_for_state_constant = 60*1

# Number of times to allow RuntimeError("short circuit to ground")
temp_reading_errors = 100
gpio_errors = 100
server_hc_errors = 100
slack_send_message_retry = 100
slack_send_message_fail_sleep = 1

# PID Proportional, Integral, and Derivative values
Pc = 3.4
Ic = 0.3
Dc = 40.0
#Pc = 8
#Ic = 0.2
#Dc = 60

Pw = 2.9
Iw = 0.3
Dw = 40.0
#Pw = 8
#Iw = 0.2
#Dw = 60

#Web/REST Server Options
port = 8080

#Slack channel to monitor
#slack_channel_type="public_channel"
slack_channel_type="im"
#slack_channel_name="general"
slack_channel_name="matthewfaw"
slack_sample_time=5

# Logging info
log_level = logging.INFO
log_maxbytes = int(1e7)
#log_level = logging.DEBUG
basedir=os.path.dirname(os.path.realpath(__file__))
log_files = {
        "main": "silvia-pi.log",
        "pid": "pid.log",
        "dummy": "dummy.log",
        "he": "heat_exchange.log",
        "schedule": "scheduler.log",
        "server": "webserver.log",
        "slack": "slack.log",
        "watchdog": "watchdog.log",
        }


# encoding=utf-8

import unicodecsv as csv
import json
import urllib2
import re
from sources import functions
from sefaria.model import *

def read_csv(filename):

    data = []
    with open(filename) as data_file:
        names = csv.DictReader(data_file, encoding='utf-8')

        for line in names:
            data.append(line)
    return data

def add_mitzva(mitzva, index):
    """
    Create an ArrayMapNode for an individual Mitzva
    :param mitzva: Dictionary with keys "English" and "Hebrew" for corresponding language names.
    :param index: Index of the mitzva to be called by Ref or url
    :return: Assembled ArrayMapNode
    """

    # check if English was added
    if re.search(u'English Translation Here', mitzva['English']):
        mitzva['English'] = u'Mitzva {}'.format(index)

    node = ArrayMapNode()
    node.wholeRef = 'Minchat Chinukh.{}'.format(index)
    node.depth = 0
    node.add_title(mitzva['English'], 'en', True)
    node.add_title(mitzva['Hebrew'], 'he', True)

    return node

def add_parsha(parsha_name, children, mitzva_node_list):
    """
    Creates a SchemaNode for a parsha with all ArrayMapNodes for mitzvot properly appended
    :param parsha_name: Name of the Parsha
    :param children: Integer or range (i.e. 3-8)
    :param mitzva_node_list: A list of ArrayMapNodes from which to select children for this Node.
    :return: SchemaNode
    """

    # determine if node has one child or many
    if children.find('-') >= 0:
        edges = [int(x) for x in children.split('-')]
        child_nodes = mitzva_node_list[edges[0]-1:edges[1]]

    else:
        child_nodes = [mitzva_node_list[int(children)-1]]

    node = SchemaNode()
    node.add_shared_term(parsha_name)
    for child in child_nodes:
        node.append(child)

    return node

def construct_alt_struct(parsha_file, mitzva_file):

    mitzvot, parashot = read_csv(mitzva_file), read_csv(parsha_file)
    mitzva_nodes= []
    root = SchemaNode()

    for index, mitzva in enumerate(mitzvot):
        mitzva_nodes.append(add_mitzva(mitzva, index+1))

    for parsha in parashot:
        root.append(add_parsha(parsha['Parsha'], parsha['Includes'], mitzva_nodes))

    return root

s = construct_alt_struct('Chinukh_by_Parsha.csv', 'Chinukh Mitzva names.csv')
data = {
        'Parsha': s.serialize()
}

raw = urllib2.urlopen('http://www.sefaria.org/api/v2/raw/index/Sefer_HaChinukh')
index = json.load(raw)
index['alt_structs'] = data
index['default_struct'] = 'Parsha'

functions.post_index(index)


import numpy as np
import cv2
import time
import warnings
def sideBySide(img1, img2, destName):
    dims1 = img1.shape
    dims2 = img2.shape
    if len(dims1) < len(dims2):
        temp = np.dstack((img1,img1,img1))
        toWrite = np.hstack((temp,img2))
    elif len(dims1) > len(dims2):
        temp = np.dstack((img2,img2,img2))
        toWrite = np.hstack((img1, temp))
    else:
        toWrite = np.hstack((img1, img2))
    cv2.imwrite(destName, toWrite)

def maxContour(contours):
    maxCont = None
    maxIdx = -1
    maxLen = -1 #cv2.arcLength(maxCont, False)
    for i, c in enumerate(contours):
        nextLen = cv2.arcLength(c, False)
        if nextLen > maxLen:
            maxCont = c
            maxIdx = i
            maxLen = nextLen
    return maxCont,maxIdx

image = cv2.imread('Test1.jpg')
ylen,xlen,d = image.shape
trimTop =ylen/2
trimRight = xlen
lines = []

#while len(lines) != 1 and (trimTop < ylen and trimRight > xlen/2):
# temp = image
# r,g,b = cv2.split(temp)
# subtract = r-g-b
# cv2.imwrite('r.jpg',r)
# cv2.imwrite('g.jpg',g)
# cv2.imwrite('b.jpg',g)
# cv2.imwrite('subtract.jpg', subtract)
# thresh = cv2.inRange(subtract, 0, 80)
# cv2.imwrite('whitewash.jpg',thresh)
# #thresh = cv2.merge((r,thresh,b))
# #thresh = cv2.medianBlur(thresh, 21)
# cont, contours, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
# h,s,v = cv2.split(temp)
# #cv2.drawContours(temp, contours, -1, (255,0,0), 3)
# canny = cv2.Canny(thresh, 100,200)
# minLineLength = 70
# maxLineGap = 60
# lines = cv2.HoughLinesP(canny,rho=1,theta=np.pi/180,threshold=10,minLineLength=minLineLength,maxLineGap=maxLineGap)
# cont, contours, hierarchy = cv2.findContours(canny, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
# cv2.drawContours(temp, contours, 4, (255,0,0), 3)
# #print len(contours)
# cv2.imwrite('tresh.jpg', thresh)
# cv2.imwrite('canny.jpg',canny)
#

############################
#Attempt via thresholding on center pixels
###################################
imageHSV = cv2.cvtColor(cv2.medianBlur(image,3), cv2.COLOR_RGB2HSV)
h,s,v = cv2.split(imageHSV)
xBox = range(xlen/2-5,xlen/2+5)
yBox = range(ylen-10, ylen)
hmean = np.mean(h[yBox,xBox])
smean = np.mean(s[yBox,xBox])
vmean = np.mean(v[yBox,xBox])

epsH = 20
epsS = 20
epsV = 50
threshH = cv2.inRange(h,hmean-epsH, hmean+epsH)
threshS = cv2.inRange(s,smean-epsS, smean+epsS)
threshV = cv2.inRange(v,vmean-epsV, vmean+epsV)
combined = (threshH) & (threshS)
combined = cv2.medianBlur(combined,5)
combined = cv2.Canny(combined,100,200)
cv2.imwrite('hThresh.jpg',threshH)
cv2.imwrite('sThresh.jpg',threshS)
cv2.imwrite('vThresh.jpg',threshV)
cv2.imwrite('combined.jpg',combined)
cont, contours, hierarchy = cv2.findContours(combined[:,xlen/2:], cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
c, i = maxContour(contours)
temp = np.zeros_like(image)
row,cols = combined.shape[:2]
[vx,vy,x,y] = cv2.fitLine(c, cv2.DIST_L2,0,0.01,0.01)
lefty = int((-x*vy/vx) + y)
righty = int(((cols-x)*vy/vx)+y)
cv2.line(image,(cols-1+xlen/2,righty),(0+xlen/2,lefty),(0,255,0),2)
#cv2.drawContours(temp, contours, -1, (255,0,0), 3)

cv2.drawContours(temp, contours, i, (255,0,0), 3)
cv2.imwrite('combinedCont.jpg',temp)

# print len(contours[319])
# longest = contours.index(max(contours, key=len))
# print len(contours)
# temp = image
# cv2.drawContours(temp, longest, -1, (255,0,0), 3)
# cv2.imwrite('temp.jpg', temp)
temp2 = np.zeros_like(image)
cv2.drawContours(temp2, contours, i, (255,0,0), 3)
sideBySide(image, temp, "HSV.jpg")

# low_black = np.array([50, 50, 50]) #if value is
# high_black = np.array([100, 255, 255])
# mask = cv2.inRange(img, low_black, high_black)
# cv2.imwrite('mask.jpg', mask)
# blurMask = cv2.GaussianBlur(mask, (7,7), 3)
# cv2.imwrite('blurMask.jpg', blurMask)
# cropped = mask[300:,500:]
# cv2.imwrite('maskCrop.jpg', cropped)
# cv2.imwrite('cannyCrop.jpg', cv2.Canny(cropped, 80,20))
# #maskRGB = cv2.cvtColor(mask, cv2.COLOR_HSV2RGB)
# #cv2.imwrite('maskRGB.jpg', maskRGB)


#
# -*- coding: utf-8 -*-
#
# Copyright (c) 2018 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# SPDX-License-Identifier: EPL-2.0
#

from __future__ import print_function

import uuid
import pytest
from conditional import conditional
from mock import call, MagicMock

from mlt.commands.deploy import DeployCommand
from test_utils.io import catch_stdout

@pytest.fixture
def sleep(patch):
    return patch('time.sleep')

@pytest.fixture
def call_logs(patch):
    return patch('log_helpers.call_logs')

@pytest.fixture
def fetch_action_arg(patch):
    return patch('files.fetch_action_arg', MagicMock(return_value='output'))

@pytest.fixture
def kube_helpers(patch):
    return patch('kubernetes_helpers')

@pytest.fixture
def json_mock(patch):
    return patch('json')

@pytest.fixture
def open_mock(patch):
    return patch('open')

# PROCESS_HELPERS MOCKS

@pytest.fixture(autouse=True)
def process_helpers_mock(patch):
    # mock every outside process generally, specific mocks below
    return patch('process_helpers')

@pytest.fixture
def run_popen_mock(patch):
    run_popen_mock = MagicMock()
    run_popen_mock.return_value.poll.return_value = 0
    return patch('process_helpers.run_popen', run_popen_mock)

@pytest.fixture
def process_helpers_run_mock(patch):
    run_mock = MagicMock(side_effect=SystemExit)
    # want to test that kubectl apply failed
    return patch('process_helpers.run', run_mock)

# END PROCESS_HELPERS MOCKS

@pytest.fixture
def progress_bar(patch):
    progress_mock = MagicMock()
    progress_mock.duration_progress.side_effect = lambda x, y, z: print(
        'Pushing ')
    return patch('progress_bar', progress_mock)

@pytest.fixture
def template(patch):
    return patch('Template')

@pytest.fixture
def verify_build(patch):
    return patch('build_helpers.verify_build')

@pytest.fixture
def verify_init(patch):
    return patch('config_helpers.load_config')

@pytest.fixture
def subprocess_mock(patch):
    return patch('check_output')

@pytest.fixture
def walk_mock(patch):
    return patch('os.walk', MagicMock(return_value=['foo', 'bar']))

@pytest.fixture
def yaml(patch):
    return patch('yaml.load')

@pytest.fixture
def get_sync_spec_mock(patch):
    return patch('sync_helpers.get_sync_spec')

@pytest.fixture
def is_custom_mock(patch):
    return patch('files.is_custom')

def deploy(no_push, skip_crd_check, interactive, extra_config_args, retries=5,
           template='test', logs=False, verbose=False, catch_exception=None):
    deploy = DeployCommand(
        {'deploy': True, '--no-push': no_push,
         '--skip-crd-check': skip_crd_check,
         '--interactive': interactive, '--retries': retries,
         '--logs': logs, '--verbose': verbose})
    deploy.config = {'name': 'app',
                     'namespace': 'namespace',
                     'template': template}

    deploy.config.update(extra_config_args)

    with catch_stdout() as caught_output:
        with conditional(catch_exception, pytest.raises(catch_exception)):
            deploy.action()
        output = caught_output.getvalue()
    return output

def verify_successful_deploy(output, did_push=True, interactive=False,
                             pod_count=2, verbose=False):
    """assert pushing, deploying, then objs created, then pushed"""
    pushing = output.find('Pushing ')
    push_skip = output.find('Skipping image push')
    deploying = output.find('Deploying ')
    inspecting = output.find('Inspect created objects by running:\n')
    pushed = output.find('Pushed app to ')
    if pod_count == 1:
        pod_connect = output.find('Connecting to pod...')
    else:
        pod_connect = output.find('watch until pods are `Running`')

    if did_push:
        # we don't print `pushing ####` stuff in `verbose` mode
        if not verbose:
            assert all(var >= 0 for var in (
                deploying, inspecting, pushing, pushed))
            assert deploying < inspecting, pushing < pushed
        else:
            assert all(var >= 0 for var in (
                deploying, inspecting, pushed))
            assert deploying < inspecting, pushed < deploying
    else:
        assert all(var == -1 for var in (pushing, pushed))
        assert all(var >= 0 for var in (deploying, inspecting, push_skip))
        assert push_skip < deploying, deploying < inspecting

    if interactive:
        assert pod_connect > inspecting

@pytest.mark.parametrize("sync_spec", [
    None, 'hello-world'])
@pytest.mark.parametrize("registry", [
    'gcr://projectfoo', 'dockerhub'])
@pytest.mark.parametrize("skip_crd_check", [
    True, False])
@pytest.mark.parametrize("logs", [
    True, False])
@pytest.mark.parametrize("verbose", [
    True, False])
def test_deploy(walk_mock, call_logs, progress_bar, run_popen_mock, open_mock,
                template, kube_helpers, verify_build,
                verify_init, fetch_action_arg, json_mock, get_sync_spec_mock,
                sync_spec, registry, skip_crd_check, logs, verbose):
    """Tests a successful deploy with and without sync_spec, 2 different
       registry types, skipping and not skipping crd_check, and log tailing
    """
    get_sync_spec_mock.return_value = sync_spec
    json_mock.load.return_value = {
        'last_remote_container': 'gcr.io/app_name:container_id',
        'last_push_duration': 0.18889}
    output = deploy(
        no_push=False, skip_crd_check=skip_crd_check,
        interactive=False, logs=logs, verbose=verbose,
        extra_config_args={'registry': registry})
    verify_successful_deploy(output, verbose=verbose)

def test_deploy_without_push(walk_mock, progress_bar, run_popen_mock,
                             open_mock, template, kube_helpers,
                             verify_build, verify_init, fetch_action_arg,
                             json_mock):
    json_mock.load.return_value = {
        'last_remote_container': 'gcr.io/app_name:container_id',
        'last_push_duration': 0.18889}
    output = deploy(
        no_push=True, skip_crd_check=True,
        interactive=False,
        extra_config_args={'registry': 'gcr://projectfoo'})
    verify_successful_deploy(output, did_push=False)

def test_deploy_interactive_one_file(walk_mock, progress_bar, run_popen_mock,
                                     open_mock, template, kube_helpers,
                                     verify_build,
                                     verify_init, fetch_action_arg, sleep,
                                     yaml, json_mock):
    walk_mock.return_value = ['foo']
    yaml.return_value = {
        'template': {'foo': 'bar'}, 'containers': [{'foo': 'bar'}]}
    json_mock.loads.return_value = {'status': {'phase': 'Running'}}
    output = deploy(
        no_push=False, skip_crd_check=True,
        interactive=True,
        extra_config_args={'registry': 'dockerhub'})
    verify_successful_deploy(output, interactive=True, pod_count=1)

    # verify that kubectl commands are specifying namespace
    for call_args in run_popen_mock.call_args_list:
        assert isinstance(call_args, type(call))
        assert isinstance(call_args[0], tuple)
        assert len(call_args[0]) > 0
        command = call_args[0][0]
        if command[0] == "kubectl":
            assert "--namespace" in command

def test_deploy_interactive_two_files(walk_mock, progress_bar, run_popen_mock,
                                      open_mock, template, kube_helpers,
                                      verify_build,
                                      verify_init, fetch_action_arg, sleep,
                                      yaml, json_mock):
    json_mock.loads.return_value = {'status': {'phase': 'Running'}}
    yaml.return_value = {
        'template': {'foo': 'bar'}, 'containers': [{'foo': 'bar'}]}
    output = deploy(
        no_push=False, skip_crd_check=True,
        interactive=True,
        extra_config_args={'registry': 'dockerhub'})
    verify_successful_deploy(output, interactive=True)

def test_deploy_interactive_pod_not_run(walk_mock, progress_bar,
                                        run_popen_mock,
                                        process_helpers_run_mock,
                                        open_mock, template, kube_helpers,
                                        verify_build,
                                        verify_init, fetch_action_arg, sleep,
                                        yaml, json_mock):
    json_mock.loads.return_value = {'status': {'phase': 'Error'}}
    yaml.return_value = {
        'template': {'foo': 'bar'}, 'containers': [{'foo': 'bar'}]}
    with pytest.raises(SystemExit):
        deploy(
            no_push=False, skip_crd_check=True,
            interactive=True,
            extra_config_args={'registry': 'dockerhub'})

def test_deploy_update_app_run_id(open_mock, json_mock):
    run_id = str(uuid.uuid4())
    json_mock_data = {
        'last_remote_container': 'gcr.io/app_name:container_id',
        'last_push_duration': 0.18889}
    json_mock.load.return_value = json_mock_data

    DeployCommand._update_app_run_id(run_id)

    assert json_mock_data['app_run_id'] == run_id

def test_image_push_error(walk_mock, progress_bar, run_popen_mock, open_mock,
                          template, kube_helpers,
                          verify_build, verify_init, fetch_action_arg,
                          json_mock):
    json_mock.load.return_value = {
        'last_remote_container': 'gcr.io/app_name:container_id',
        'last_push_duration': 0.18889}

    # setup mock to induce and error during the deploy
    run_popen_mock.return_value.poll.return_value = 1
    output_str = "normal output..."
    error_str = "error message..."
    build_output = MagicMock()
    build_output.decode.return_value = output_str
    error_output = MagicMock()
    error_output.decode.return_value = error_str
    run_popen_mock.return_value.communicate.return_value = (build_output,
                                                            error_output)

    output = deploy(
        no_push=False, skip_crd_check=True, interactive=False,
        extra_config_args={'registry': 'gcr://projectfoo'},
        catch_exception=SystemExit)

    # assert that we got the normal output, followed by the error message
    output_location = output.find(output_str)
    error_location = output.find(error_str)
    assert all(var >= 0 for var in (output_location, error_location))
    assert output_location < error_location

def test_no_image_found_error(walk_mock, progress_bar, run_popen_mock,
                              open_mock, template, kube_helpers,
                              verify_build, verify_init, fetch_action_arg,
                              json_mock):
    """if we don't have remote_container_name during deploy_new_container
       then throw ValueError
    """
    get_sync_spec_mock.return_value = None
    fetch_action_arg.side_effect = [None, None, None]
    json_mock.load.return_value = {
        'last_remote_container': 'gcr.io/app_name:container_id',
        'last_push_duration': 0.18889}
    deploy(
        no_push=True, skip_crd_check=True, interactive=False,
        extra_config_args={'registry': 'gcr://projectfoo'},
        catch_exception=ValueError)

def test_deploy_custom_deploy(walk_mock, progress_bar, run_popen_mock,
                              open_mock, template, kube_helpers,
                              subprocess_mock, verify_build, is_custom_mock,
                              verify_init, fetch_action_arg, json_mock, ):
    json_mock.load.return_value = {
        'last_remote_container': 'gcr.io/app_name:container_id',
        'last_push_duration': 0.18889}
    expected_output = "Updating configmaps\n" \
                      "Creating non-existent configmaps\n" \
                      "Updating services\n" \
                      "Creating non-existent services\n" \
                      "Updating serviceaccounts\n" \
                      "Creating non-existent serviceaccounts\n" \
                      "Updating clusterrolebindings\n" \
                      "Creating non-existent clusterrolebindings\n" \
                      "Updating pods\n" \
                      "Creating non-existent pods\n"
    subprocess_mock.return_value.decode.return_value = expected_output
    is_custom_mock.return_value = True
    output = deploy(
        no_push=False, skip_crd_check=True,
        interactive=False,
        extra_config_args={'registry': 'gcr://projectfoo',
                           "template_parameters": {
                               "gpus": 0,
                               "num_workers": 1}})
    verify_successful_deploy(output)
    assert expected_output in output

def test_deploy_custom_deploy_interactive(walk_mock, progress_bar,
                                          run_popen_mock, open_mock, template,
                                          kube_helpers, subprocess_mock,
                                          verify_build, is_custom_mock,
                                          verify_init, fetch_action_arg,
                                          json_mock, yaml):
    json_mock.load.return_value = {
        'last_remote_container': 'gcr.io/app_name:container_id',
        'last_push_duration': 0.18889}
    is_custom_mock.return_value = True
    json_mock.loads.return_value = {'status': {'phase': 'Running'}}
    yaml.return_value = {
        'template': {'foo': 'bar'}, 'containers': [{'foo': 'bar'}]}
    output = deploy(
        no_push=False, skip_crd_check=True,
        interactive=True,
        extra_config_args={'registry': 'gcr://projectfoo',
                           "template_parameters": {
                               "gpus": 0,
                               "num_workers": 1}})
    verify_successful_deploy(output, interactive=True)


from tabulate import tabulate
import numpy as np
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import tensorflow as tf
tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)
from sklearn import metrics
import random
from random import randint
import time
import os

def load_X(X_path):
    file = open(X_path, 'r')
    X_ = np.array( # _ is for object
        [elem for elem in [row.split(',') for row in file]],  # parsing the text file
        dtype=np.float32
    )
    file.close()
    blocks = int(len(X_) / n_steps)
    X_ = np.array(np.split(X_,blocks)) # shape will be arrays of length 33
    return X_

# Load the networks outputs
def load_y(y_path):
    file = open(y_path, 'r')
    y_ = np.array(
        [elem for elem in [row.replace('  ', ' ').strip().split(' ') for row in file]],
        dtype=np.int32
    )
    file.close()
    # for 0-based indexing
    return y_ - 1

# Preparing dataset:
# Useful Constants
# Output classes to learn how to classify
LABELS = [
    "JUMPING",
    "JUMPING_JACKS",
    "BOXING",
    "WAVING_2HANDS",
    "WAVING_1HAND",
    "CLAPPING_HANDS"
]
DATASET_PATH = "data/HAR_pose_activities/database/"
X_train_path = DATASET_PATH + "X_train.txt"
X_test_path = DATASET_PATH + "X_test.txt"
y_train_path = DATASET_PATH + "Y_train.txt"
y_test_path = DATASET_PATH + "Y_test.txt"
X_val_path = DATASET_PATH + "X_val.txt"
n_steps = 32 # 32 timesteps per series
# A training step is one gradient update.
X_train = load_X(X_train_path)
X_test = load_X(X_test_path)
y_train = load_y(y_train_path)
y_test = load_y(y_test_path)
X_val = load_X(X_val_path)
# ===================== Parameters ======================= #
# Input Data
training_data_count = len(X_train)  # 4519 training series (with 50% overlap between each serie)
test_data_count = len(X_test)  # 1197 test series
n_input = len(X_train[0][0])  # num input parameters per timestep
n_hidden = 34 # Hidden layer num of features
n_classes = 6

#updated for learning-rate decay
# calculated as: decayed_learning_rate = learning_rate * decay_rate ^ (global_step / decay_steps)
decaying_learning_rate = True
learning_rate = 0.0025 #used if decaying_learning_rate set to False
init_learning_rate = 0.0025
decay_rate = 0.96 #the base of the exponential in the decay
decay_steps = 100000 #used in decay every 60000 steps with a base of 0.96
global_step = tf.Variable(0, trainable=False)
lambda_loss_amount = 0.0015
training_iters = training_data_count*1  # Loop 300 times on the dataset, ie 300 epochs
batch_size = 256
display_iter = batch_size*8  # To show test set accuracy during training

# Load the networks inputs

def placeholders_for_input_data():
    '''
        placeholder
        Assigning variable to memory and defining its type and dimensions ie row: none and column and depth
        an example would be an RBG image with [k, m, 3] to accomodate for k x m resolution and 3 colour channel for the pixel
        Note: Differs from Variable as variables are meant to be trained and changed over time and must have an initial often random number
        whereas placeholder can have no initial value and once assigned does not change
    '''
    x = tf.placeholder(tf.float32, [None, n_steps, n_input], name = "x")
    y = tf.placeholder(tf.float32, [None, n_classes], name = "y")
    return x,y

def assign_random_weights_and_biases_as_variables():
    '''
        Variable
        Variables store tensors. Specific ops allow you to read and modify the values of this tensor.
        TensorFlow variables use lazy loading. Therefore, if you want to see the value of var, you have to create a session
        print(sess.run(var))
        When you train a model, you use variables to hold and update parameters.
        Here we are assigning random weights and biases to start with, by default trainable = True
        and it is added to a global variable somewhere
        The get_variable() function gets an existing variable with the given parameters or creates a new one.
    '''
    weights = {
        'hidden': tf.Variable(tf.random_normal([n_input, n_hidden]), name= "weights_hidden"),
        'out': tf.Variable(tf.random_normal([n_hidden, n_classes], mean=1.0, name= "weights_out"))
    }
    biases = {
        'hidden': tf.Variable(tf.random_normal([n_hidden]), name = "biases_hidden"),
        'out': tf.Variable(tf.random_normal([n_classes]), name = "biases_out")
    }
    return weights, biases

# Utility functions for training:
def LSTM_RNN(_X, _weights, _biases):
    # model architecture based on "guillaume-chevalier" and "aymericdamien" under the MIT license.
    _X = tf.transpose(_X, [1, 0, 2])  # permute n_steps and batch_size
    _X = tf.reshape(_X, [-1, n_input])
    # Rectifies Linear Unit activation function used
    _X = tf.nn.relu(tf.matmul(_X, _weights['hidden']) + _biases['hidden'])
    # Split data because rnn cell needs a list of inputs for the RNN inner loop
    _X = tf.split(_X, n_steps, 0)

    # Define two stacked LSTM cells (two recurrent layers deep) with tensorflow
    lstm_cell_1 = tf.contrib.rnn.BasicLSTMCell(n_hidden, forget_bias=1.0, state_is_tuple=True)
    lstm_cell_2 = tf.contrib.rnn.BasicLSTMCell(n_hidden, forget_bias=1.0, state_is_tuple=True)
    lstm_cells = tf.contrib.rnn.MultiRNNCell([lstm_cell_1, lstm_cell_2], state_is_tuple=True)
    outputs, states = tf.contrib.rnn.static_rnn(lstm_cells, _X, dtype=tf.float32)

    # A single output is produced, in style of "many to one" classifier, refer to http://karpathy.github.io/2015/05/21/rnn-effectiveness/ for details
    lstm_last_output = outputs[-1]
    model = tf.matmul(lstm_last_output, _weights['out']) + _biases['out'] # y = W*x + B
    model = tf.identity(model, name="model")
    # Linear activation
    return model

def cost_function():
    '''
        Loss
        Loss is the value the model aims to reduce. If you recall we test the final layer (output) against
        ground truth and the difference squared was the backpropogation input that would correct the network
        by changing weights and biases proportionately
        L1 Loss Function is mean absolute deviation(MAD) |y2-y1|/n
        L2 Loss Function is mean squared error(MSE) (y2-y1)^2/n
        L2 would be bad with outliers as the difference is squared so the corrective movement will be large
        Softmax
        Softmax is a normalization function that reduces input from 0 to 1.
        It is commonly used as standard normalisation just takes into account proportion (1,2) --> (0.33,0.66)
        but softmax also takes size of data into account so (10,20) --> (0.000045, 0.99) which is useful in machine learning
        as outputs are higher when for eg the picture is clearer and more neurons are fired
        Cross entropy
        Cross entropy is another loss function that computes the loss based on the log difference between predicted value and
        true prediction value
        So in this case softmax outputs 0 to 1 and cross entropy computes the loss using that value.
        Logits just means unnormalized log probabilities
        We also add L2 Regularization it penalises the loss function and reduces the weights for
        It works on assumption that smaller weights generate simpler model and thus helps avoid overfitting.

    '''
    l2_regularization = lambda_loss_amount * sum(tf.nn.l2_loss(tf_var) for tf_var in tf.trainable_variables()) # L2 loss prevents this overkill neural network to overfit the data
    cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=model)) + l2_regularization # Softmax loss
    return cost

def get_optimizer():
    '''
        Given an algorithm f(x), an optimization algorithm help in either minimizing or maximizing the value of f(x)
        We use the Adam optimiser here instead of stochastic gradient descent
    '''
    optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost,global_step=global_step) # Adam Optimizer
    return optimizer

def get_accuracy(model):
    '''
        If equal return true
        Argmax returns index of biggest element in tensor, argmax(array,axis) axis of 1 is columns, axis of 0 is rows
        For example tensor returned is [3,4,1] then equal() is run and produces tensor of (1,1,1) if all is equal
    '''
    correct_pred = tf.equal(tf.argmax(model,1), tf.argmax(y,1))
    accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) # cast changes datatype
    return accuracy

def extract_batch_size(_train, _labels, _unsampled, batch_size):
    # Fetch a "batch_size" amount of data and labels from "(X|y)_train" data.
    # Elements of each batch are chosen randomly, without replacement, from X_train with corresponding label from Y_train
    # unsampled_indices keeps track of sampled data ensuring non-replacement. Resets when remaining datapoints < batch_size
    shape = list(_train.shape)
    shape[0] = batch_size
    batch_s = np.empty(shape)
    batch_labels = np.empty((batch_size,1))
    _unsampled = list(_unsampled)
    # TODO replace this with library function
    for i in range(batch_size):
        index = random.choice(_unsampled)
        batch_s[i] = _train[index]
        batch_labels[i] = _labels[index]
        _unsampled.remove(index) # remove indexes to be sampled
    return batch_s, batch_labels, _unsampled

def train_model():
    test_losses = []
    test_accuracies = []
    train_losses = []
    train_accuracies = []
    sess = tf.InteractiveSession(config=tf.ConfigProto(log_device_placement=True))
    # interactive session allows you to stop and evaluate and change the graph at any time
    # this delayed execution is one of the advantages of tensorflow
    init = tf.global_variables_initializer()
    # Variable need to be "initialised" in tensorflow, creation is not enough
    sess.run(init)
    # Perform Training steps with "batch_size" amount of data at each loop.
    # Elements of each batch are chosen randomly, without replacement, from X_train,
    # restarting when remaining datapoints < batch_size
    step = 1
    unsampled_indices = range(0,len(X_train))

    while step * batch_size <= training_iters:
        #print (sess.run(learning_rate)) #decaying learning rate
        #print (sess.run(global_step)) # global number of iterations
        if len(unsampled_indices) < batch_size:
            unsampled_indices = range(0,len(X_train))
        batch_xs, raw_labels, unsampled_indicies = extract_batch_size(X_train, y_train, unsampled_indices, batch_size)
        # print(raw_labels) # in index 1-6
        batch_ys = one_hot(raw_labels)
        # print(batch_ys) # in array of arrays of 0s with 1 at the index
        # check that encoded output is same length as num_classes, if not, pad it
        # (?) must i encode the labels
        if len(batch_ys[0]) < n_classes:
            temp_ys = np.zeros((batch_size, n_classes)) # (256, 6)
            temp_ys[:batch_ys.shape[0],:batch_ys.shape[1]] = batch_ys
            # [:] is slice operator [:,:] is slice row, slice column, replaces those values with batch_ys
            batch_ys = temp_ys
        # Fit training using batch data
        '''
            session.run(fetches=,feed_dict=)
            fetches are operations, the things that need to be computed, so for example if W is a constant and x a Variable,
            Wx = tf.multiply(W,x) is an operation/node that is to be computed. So sess.run(Wx, feed_dict={x: [3,33]})
            will return Wx.
            underscore is the optimizer but it is not needed.
            cost function returns the loss
            accuracy returns the accuracy
            apparently these functions(nodes) are global or at least appended to the graph already
            Inside the run function we are computing 3 operations, optimizer, cost and accuracy
            cost computes the loss based on the model and ground truth (can replace accuracy?)
            optimizer works to reduce cost
            accuracy evaluates the accuracy based on the difference between predicted value and ground truth
            feed dict provides input to the placeholders x and y
            W and B gets changed because first cost function produces loss from the last layer of the model y = Wx+B
            Then that is fed into the optimizer which reduces the loss by changing the weights and bias from the last layer of the model
            and then it backpropogates until all the nodes are visited through the concept of automatic differentiation which simply
            and probably inaccurately explained is that the gradient of each sequential small operation like + or - can be obtained
            and combined via chain rule.
        '''
        _, loss, acc = sess.run(
            [optimizer, cost, accuracy],
            feed_dict={
                # y = W*x + B
                x: batch_xs,
                y: batch_ys
            }
        )
        train_losses.append(loss)
        train_accuracies.append(acc)

        # Evaluate network only at some steps for faster training:
        if (step*batch_size % display_iter == 0) or (step == 1) or (step * batch_size > training_iters):
            # To not spam console, show training accuracy/loss in this "if"
            print("Iter no." + str(step) + ":  Learning rate = " + "{:.6f}".format(sess.run(learning_rate)) + ":   Batch Loss = " + "{:.6f}".format(loss) + ", Accuracy = {}".format(acc))
            # Evaluation on the test set (no learning made here - just evaluation for diagnosis)
            loss, acc = sess.run(
                [cost, accuracy],
                feed_dict={
                    x: X_test,
                    y: one_hot(y_test)
                }
            )
            test_losses.append(loss)
            test_accuracies.append(acc)
            print("Performance on test set:" + "Batch Loss = {}".format(loss) + ", Accuracy = {}".format(acc))
        step += 1
    print("Optimization Finished!")
    return sess, test_accuracies, test_losses, train_accuracies, train_losses

def one_hot(y_):
    # One hot encoding of the network outputs
    # e.g.: [[5], [0], [3]] --> [[0, 0, 0, 0, 0, 1], [1, 0, 0, 0, 0, 0], [0, 0, 0, 1, 0, 0]]
    y_ = y_.reshape(len(y_))
    n_values = int(np.max(y_)) + 1
    return np.eye(n_values)[np.array(y_, dtype=np.int32)]  # Returns FLOATS

def plot_training_graph(train_accuracies,train_losses,width,height):
    font = {
        'family' : 'Bitstream Vera Sans',
        'weight' : 'bold',
        'size'   : 18
    }
    matplotlib.rc('font', **font)
    plt.figure(figsize=(width, height))
    indep_train_axis = np.array(range(batch_size, (len(train_losses)+1)*batch_size, batch_size))
    #plt.plot(indep_train_axis, np.array(train_losses),     "b--", label="Train losses")
    plt.plot(indep_train_axis, np.array(train_accuracies), "g--", label="Train accuracies")
    indep_test_axis = np.append(
        np.array(range(batch_size, len(test_losses)*display_iter, display_iter)[:-1]),
        [training_iters]
    )
    #plt.plot(indep_test_axis, np.array(test_losses), "b-", linewidth=2.0, label="Test losses")
    plt.plot(indep_test_axis, np.array(test_accuracies), "b-", linewidth=2.0, label="Test accuracies")
    plt.title("Training session's Accuracy over Iterations")
    plt.legend(loc='lower right', shadow=True)
    plt.ylabel('Training Accuracy')
    plt.xlabel('Training Iteration')
    plt.savefig("graph.png")

def plot_confusion_matrix(data, width, height):
    #cmap = plt.cm.Blues
    plt.title("Confusion matrix \n(normalised to % of total test data)")
    plt.matshow(normalised_confusion_matrix)
    plt.colorbar()
    tick_marks = np.arange(n_classes)
    plt.xticks(tick_marks, LABELS, rotation=90)
    plt.yticks(tick_marks, LABELS)
    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')
    plt.savefig("confusion_matrix.png")

def predict(sess):
    # Now, you run this with fine-tuning data in sess.run()
    # preds = sess.run(
    #     [model],
    #     feed_dict={
    #         x: X_val
    #     }
    # )
    prediction=tf.argmax(model,1)
    prediction = prediction.eval(feed_dict={x: X_val},session=sess)
    prediction = LABELS[prediction[0]]
    print ("Prediction is: {}".format(prediction))

if __name__ == "__main__":
    # Replace labelled classes with random classes to train on
    # for i in range(len(y_train)):
    #     y_train[i] = randint(0, 5)
    print(tabulate([[X_train.shape, y_test.shape, np.mean(X_test), np.std(X_test)]],
    headers=['X shape', 'y shape','X mean', 'X standard deviation' ]))
    # with tf.device('/gpu:0'):
    # Graph input/output
    x,y = placeholders_for_input_data()
    # Graph weights
    weights, biases = assign_random_weights_and_biases_as_variables()
    model = LSTM_RNN(x, weights, biases)
    # Loss, optimizer and evaluation
    cost = cost_function()
    if decaying_learning_rate:
        learning_rate = tf.train.exponential_decay(init_learning_rate, global_step*batch_size, decay_steps, decay_rate, staircase=True)
    optimizer = get_optimizer()
    accuracy = get_accuracy(model)
    # Train the network:
    time_start = time.time()
    sess, test_accuracies, test_losses, train_accuracies, train_losses = train_model()
    time_stop = time.time()
    print("Total time:  {}".format(time_stop - time_start))
    # ======== Save model ========== #
    saver = tf.train.Saver()
    saver.save(sess, 'model_steps/model.ckpt')
    # Accuracy for test data
    one_hot_predictions, accuracy, final_loss = sess.run(
        [model, accuracy, cost],
        feed_dict={
            x: X_test,
            y: one_hot(y_test)
        }
    )
    test_losses.append(final_loss)
    test_accuracies.append(accuracy)
    # Results
    print("Final result: " + "Batch Loss = {}".format(final_loss) + ", Accuracy = {}".format(accuracy))
    predictions = one_hot_predictions.argmax(1)
    print(metrics.classification_report(y_test, predictions, target_names=LABELS))
    # Plot Results:
    plot_training_graph(train_accuracies, train_losses,16,16)
    confusion_matrix = metrics.confusion_matrix(y_test, predictions)
    normalised_confusion_matrix = np.array(confusion_matrix, dtype=np.float32)/np.sum(confusion_matrix)*100
    plot_confusion_matrix(normalised_confusion_matrix, 16, 16)
    prediction=tf.argmax(model,1)
    predict(sess) # must feed in the same sess to get the same result as above
    sess.close()


import sys
import numpy as np
from scipy import sparse
from time import time

def sim(u1, u2, sig):
	"""
	Measure and return the similarity between the signatures (sig) of two users (u1, u2).
	"""
	similarity = np.sum(np.equal(sig[:,u1],sig[:,u2])).astype(float)/len(sig[:,u1])
	return similarity

def jsim(u1, u2, sm):
    """Calculate and return the Jaccard similarity between two users
    (u1 and u2) with the sparse matrix sm """
    a = set(sm.indices[sm.indptr[u1]:sm.indptr[u1+1]])
    b = set(sm.indices[sm.indptr[u2]:sm.indptr[u2+1]])
    c = a.intersection(b)
    return float(len(c)) / (len(a) + len(b) - len(c))

def band_to_bucket(band):
	"""
	map each column in a band to buckets as a dictionary,
	return the list of items within the same bucket
	"""
	buckets={}
	for c in range(band.shape[1]):
		buckets.setdefault(tuple(band[:,c]),[]).append(c)
	return [locs for key,locs in buckets.items() if len(locs)>1]

def main():
	before=time()
	#read 2 arguments: random seed and path of the data
	seeds = int(sys.argv[1])
	datafile = sys.argv[2]

	print("Loading the data......")
	data = np.load(datafile)

	#construct sparse matrix of (N_user, N_movie)
	N_usr = np.max(data[:,0])+1
	N_mov = np.max(data[:,1])+1
	V = np.ones(data.shape[0])
	A = sparse.csc_matrix((V,(data[:,1],data[:,0])),shape=(N_mov,N_usr))

	#size of signature
	N_sig = 100
	print("Generating signature......")
	sig = np.zeros((N_sig,N_usr)).astype('int32')
	for k in range(N_sig):
		#generate random permutation
		np.random.seed(seeds+k)
		A_permute = np.random.permutation(N_mov)
		#apply permutation to the sparse matrix
		A_new = A[A_permute,:]
		#find the index of the first non-zero element in each column
		for j in range(N_usr):
			ar = A_new.indices[A_new.indptr[j]:A_new.indptr[j + 1]].min()
			sig[k,j] = ar

	print("Signature is generated.")

	b=20
	r=5

	#split the signature matrix into b=20 bands with r=5 rows each.
	subarrays = np.split(sig, range(r, N_sig, r))

	print("Generating candidates......")
	#'sim_list' containing the priliminary similar pairs by comparing the signature similarity
	sim_list=[]
	for band in subarrays:
		pairs = band_to_bucket(band.astype(int))
		for bucket in pairs:
			#ignore the buckets with size larger than a threshold 200.
			if len(bucket)<200:
				sim_list+=[(u1,u2) for u1 in bucket for u2 in bucket \
				if u1<u2 if sim(u1,u2,sig)>0.5]
	sim_list=set(sim_list)

	print("Sifting the candidates to find the true similar pairs......")
	#'sim_list' containing the true similar pairs by comparing the jaccard similarity of full data
	user_pair_list=[]
	for pair in sim_list:
	    # if jaccard similarity > 0.5, add to the list
	    similarity = jsim(pair[0], pair[1], A)
	    if similarity > 0.5:
	        user_pair_list.append((pair[0], pair[1], similarity))

	N_pairs = len(user_pair_list)
	print("Number of pairs found: {0}".format(N_pairs))
	print("Number of pairs per minute: {0:.1f}".format(N_pairs/((time()-before)/60.0)))
	user_pair_list.sort()
	outfile = 'user-pairs.txt'
	print("Saving to file......")
	np.savetxt(outfile, user_pair_list, fmt="%d,%d,%.6f", delimiter=',')

if __name__ == '__main__':
	main()


from nltk.sentiment.vader import SentimentIntensityAnalyzer
import nltk
nltk.download('vader_lexicon')

def sentiment_analyzer(sentiment_text):
    score = SentimentIntensityAnalyzer().polarity_scores(sentiment_text)
    neg = score['neg']
    pos = score['pos']
    neu = score['neu']
    if pos > neg:
        polarity = "Positive Sentiment"
    elif pos < neg:
        polarity = "Negative Sentiment"
    else:
        polarity = "Neutral Vibe"
    return [pos, neg, neu, polarity]


import requests
import json

API_KEY = "KEY"
class YandexTranslator:
    @staticmethod
    def translate(*args):
        url = "https://translate.yandex.net/api/v1.5/tr.json/translate?key="
        if not len(args) > 1:
            raise Exception("Parameters mismatch.")
        text = args[0]
        if len(args) == 2:
            destLang = args[1]
            sourceLang = detectLanguage(text)
        else:
            sourceLang = args[1]
            destLang = args[2]

        return requests.get(url + API_KEY + "&text=" + text + "&lang=" + sourceLang + "-" + destLang)

    @staticmethod
    def detectLanguage(text):
        langDetectUrl = "https://translate.yandex.net/api/v1.5/tr.json/detect?key="
        response = (requests.get(langDetectUrl + API_KEY + "&text=" + text)).json()
        return response["lang"]


import json

from flask import Flask, render_template, request, redirect
from flask.ext.wtf.form import _is_hidden
from elasticsearch import Elasticsearch

import forms

app = Flask(__name__)
app.config.from_pyfile('config.py')

@app.route('/')
def index():
    item_search_form = forms.ItemSearchForm()
    api_form = forms.ExileToolsAPIInputForm()
    return render_template(
        'index.html', title='Indexile', form=item_search_form, is_hidden=_is_hidden, api_form=api_form)

@app.route('/search')
def search():
    return str(request.args)

@app.route('/api_search', methods=['POST'])
def api_search():
    api_form = forms.ExileToolsAPIInputForm()
    if api_form.validate_on_submit():
        es = Elasticsearch(
            [{'host': 'api.exiletools.com', 'port': 80, 'http_auth': 'apikey:'+app.config['EXILETOOLS_APIKEY']}])
        res = es.search(index='index', body=json.loads(api_form.json.data))
        return render_template('search_results.html', results=res, isinstance=isinstance)
    else:
        return redirect('/')

if __name__ == '__main__':
    app.run(debug=True)


# -*- coding: utf-8 -*-
import logging
logger = logging.getLogger(__name__)

def _patch_diazo():
    try:
        from plone.app.theming.transform import ThemeTransform
        from perfmetrics import Metric
        from zperfmetrics import ZMetric
    except ImportError:
        logger.info('No Plone Diazo patches for zperfmetrics')
        return

    # patch to measure plone.app.theming
    logger.info('Activating Plone Diazo patches for zperfmetrics')
    ThemeTransform.setupTransform = Metric(
        stat='diazo.setup',
        method=True,
    )(
        ThemeTransform.setupTransform
    )
    ThemeTransform.transformIterable = ZMetric(
        stat='diazo.transform',
        method=True
    )(
        ThemeTransform.transformIterable
    )

def initialize(context):
    _patch_diazo()


from django.core.management.base import BaseCommand
from corehq.apps.domain.models import Domain
from corehq.feature_previews import LOCATIONS
from corehq.toggles import NAMESPACE_DOMAIN
from toggle.shortcuts import update_toggle_cache, namespaced_item
from toggle.models import Toggle

class Command(BaseCommand):
    def handle(self, *args, **options):
        domains = Domain.get_all()

        for domain in domains:
            if domain.commtrack_enabled:
                toggle = Toggle.get(LOCATIONS.slug)
                toggle_user_key = namespaced_item(domain.name, NAMESPACE_DOMAIN)

                if toggle_user_key not in toggle.enabled_users:
                    toggle.enabled_users.append(toggle_user_key)
                    toggle.save()
                    update_toggle_cache(LOCATIONS.slug, toggle_user_key, True)

                if not domain.locations_enabled:
                    domain.locations_enabled = True
                    domain.save()


"""Example of an instruction paired with agent actions."""
from __future__ import annotations

from typing import TYPE_CHECKING

import numpy as np

from agent import util
from agent.data import gameplay_action
from agent.data import partial_observation
from agent.environment import agent_actions
from agent.environment import environment_objects
from agent.environment import state_delta
from agent.environment import terrain
from agent.environment import util as environment_util

if TYPE_CHECKING:
    from typing import Dict, List, Optional, Set, Tuple
    from agent.data import cereal_bar_game
    from agent.environment import card
    from agent.environment import position

def get_changed_cards_along_trajectory(state_sequence: List[state_delta.StateDelta]) -> List[card.Card]:
    unique_visitation_sequence: List[Tuple[position.Position, bool]] = list()
    original_position: position.Position = state_sequence[0].follower.get_position()
    original_card_positions: List[position.Position] = [original_card.get_position() for original_card in
                                                        state_sequence[0].cards]
    has_moved: bool = False

    for i, delta in enumerate(state_sequence[1:]):
        follower_position = delta.follower.get_position()
        if follower_position == original_position and not has_moved:
            continue
        has_moved = True
        # Only keep track of the position if it wasn't the last one it was in
        if not unique_visitation_sequence or unique_visitation_sequence[-1][0] != follower_position:
            # Also check the previous state information to see if there was a card in that location when it
            # stepped there (check the previous information because otherwise this won't count cards that
            # disappeared when the agent took a step). Also can't just check the original state because
            # the agent could return to a location that it had removed a card from.

            # this is previous because i will be starting from the 1th example.
            unique_visitation_sequence.append(
                (follower_position,
                 follower_position in [state_card.get_position() for state_card in state_sequence[i].cards]))

    # Filter by states where there was a card touched, and specifically if it was a card that was only in the
    # original state (if a card appeared later, the instruction couldn't have referred to it, so it's not a
    # required card to touch).
    unique_card_visitations: List[position.Position] = \
        [pos[0] for pos in unique_visitation_sequence if pos[1] and pos[0] in original_card_positions]

    # Iterates through all unique positions reached which have a card, and counts how many times they have
    # been reached.
    card_pos_visitation_dict: Dict[position.Position, int] = \
        {pos2: len([pos for pos in unique_card_visitations if pos == pos2])
         for pos2 in set(unique_card_visitations)}

    # Filters this by odd visitations -- meaning that the selection of the card should have changed by the end
    # of the instruction.
    reached_card_positions: Set[position.Position] = \
        {pos for pos, count in card_pos_visitation_dict.items() if count % 2 == 1}

    reached_cards: List[card.Card] = list()
    for initial_card in state_sequence[0].cards:
        if initial_card.get_position() in reached_card_positions:
            reached_cards.append(initial_card)

    return reached_cards

class InstructionExample:
    def __init__(self,
                 instruction: List[str],
                 target_action_sequence: List[Tuple[state_delta.StateDelta, agent_actions.AgentAction,
                                                    partial_observation.PartialObservation]],
                 paired_game: cereal_bar_game.CerealBarGame,
                 example_idx_in_interaction: int,
                 leader_actions: List[List[gameplay_action.GameplayAction]],
                 index_of_first_leader_turn: int,
                 sets_made_during_instruction: List[Tuple[List[card.Card], List[card.Card]]],
                 number_of_steps_in_first_turn: int,
                 number_of_instructions_when_starting: int):
        self._instruction: List[str] = instruction
        self._target_action_sequence: List[
            Tuple[state_delta.StateDelta, agent_actions.AgentAction,
                  partial_observation.PartialObservation]] = target_action_sequence
        self._paired_game: cereal_bar_game.CerealBarGame = paired_game
        self._example_idx_in_interaction: int = example_idx_in_interaction
        self._leader_actions: List[List[gameplay_action.GameplayAction]] = leader_actions
        self._index_of_first_leader_turn: int = index_of_first_leader_turn
        self._sets_made_during_instruction: List[Tuple[List[card.Card], List[card.Card]]] = sets_made_during_instruction
        self._number_of_steps_in_first_turn: int = number_of_steps_in_first_turn
        self._number_of_instructions_when_starting: int = number_of_instructions_when_starting

        self._static_indices = None

    def hash_representation(self) -> str:
        str_hash: str = self.get_id() + '/' + \
                        ' '.join([str(action) for action in self.get_action_sequence()]) + '/' + \
                        str(self.get_initial_state().follower.get_position()) + '/' + \
                        str(self.get_initial_state().follower.get_rotation()) + '/'
        for initial_card in self.get_initial_state().cards:
            str_hash += str(initial_card) + '/'
        return str_hash

    def get_seed(self) -> int:
        return self._paired_game.get_seed()

    def get_id(self) -> str:
        return self._paired_game.get_id() + '-' + str(self._example_idx_in_interaction)

    def get_instruction(self) -> List[str]:
        return self._instruction

    def get_action_sequence(self) -> List[str]:
        return [str(x[1]) for x in self._target_action_sequence]

    def get_state_deltas(self) -> List[state_delta.StateDelta]:
        return [x[0] for x in self._target_action_sequence]

    def get_initial_cards(self) -> List[card.Card]:
        return self.get_initial_state().cards

    def get_initial_state(self) -> state_delta.StateDelta:
        return self._target_action_sequence[0][0]

    def get_static_indices(self, state_representation) -> np.array:
        if self._static_indices is None:
            self._static_indices = state_representation.static_indices(self)
        return self._static_indices

    def get_objects(self) -> List[environment_objects.EnvironmentObject]:
        return self._paired_game.get_objects()

    def get_obstacle_positions(self) -> List[position.Position]:
        obstacle_positions: List[position.Position] = list()
        for ter, hexp in self.get_hexes():
            if ter in terrain.OBSTACLE_TERRAINS:
                obstacle_positions.append(hexp)
        for obj in self.get_objects():
            assert obj.get_type() != environment_objects.ObjectType.CARD
            obstacle_positions.append(obj.get_position())
        return obstacle_positions

    def get_hexes(self) -> List[Tuple[terrain.Terrain, position.Position]]:
        return self._paired_game.get_hexes()

    def get_visited_positions(self, include_start: bool = True, start_idx: int = 0) -> List[position.Position]:
        """Gets an ordered list of positions visited by the agent along the gold trajectory.

        Args:
            include_start: Whether to include the start position in the list.
            start_idx: The action index to start from.

        """
        if include_start:
            return list(set([delta.follower.get_position() for delta in self.get_state_deltas()[start_idx:]]))

        state: state_delta.StateDelta = self.get_state_deltas()[start_idx]
        original_position: position.Position = state.follower.get_position()
        pos_list: List[position.Position] = []
        has_moved: bool = False
        for delta in self.get_state_deltas()[start_idx:]:
            current_position = delta.follower.get_position()
            if current_position == original_position and not has_moved:
                continue
            else:
                has_moved = True
            pos_list.append(current_position)

        return list(set(pos_list))

    def get_first_partial_observation(self) -> partial_observation.PartialObservation:
        return self._target_action_sequence[0][2]

    def get_partial_observations(self) -> List[partial_observation.PartialObservation]:
        return [x[2] for x in self._target_action_sequence]

    def get_leader_actions(self,
                           limit_to_instruction: bool = True,
                           use_all_actions: bool = False) -> List[List[gameplay_action.GameplayAction]]:
        """Returns the leader actions aligning to this instruction.

        Args:
            limit_to_instruction: Whether to only return actions that occurred during the execution of this instruction.
            use_all_actions: TODO: What does this mean?
        """
        if limit_to_instruction:
            return self._leader_actions
        actions_from_game: List[List[gameplay_action.GameplayAction]] = self._paired_game.get_leader_actions(
            first_turn=self._index_of_first_leader_turn,
            use_all_actions=use_all_actions)
        if use_all_actions:
            return actions_from_game

        for action1, action2 in zip(self._leader_actions, actions_from_game):
            if action1 != action2:
                raise ValueError('Actions not the same!')
        return actions_from_game

    def get_expected_sets(self) -> List[Tuple[List[card.Card], List[card.Card]]]:
        """Gets the sets that should be made during this instruction and afterwards."""
        return self._paired_game.get_expected_sets(first_instruction=self._example_idx_in_interaction)

    def get_number_of_moves_in_first_turn(self) -> int:
        return self._number_of_steps_in_first_turn

    def get_number_of_instructions_when_starting(self) -> int:
        return self._number_of_instructions_when_starting

    def get_final_state(self) -> state_delta.StateDelta:
        if self._target_action_sequence[-1][1] != agent_actions.AgentAction.STOP:
            raise ValueError('Final action was not STOP')

        # The final state is the prior state for the last action, which is the STOP action.
        return self._target_action_sequence[-1][0]

    def get_touched_cards(self,
                          start_idx: int = 0,
                          include_start_position: bool = False,
                          allow_duplicates: bool = True) -> List[card.Card]:
        """Gets all cards touched along the gold trajectory.

        Args:
            start_idx: The first index along the trajectory to consider when computing which cards were touched.
            include_start_position: Whether to include any cards that are in the initial position.
            allow_duplicates: TODO: Not sure what this setting does actually.

        """
        if allow_duplicates:
            state: state_delta.StateDelta = self.get_state_deltas()[start_idx]
            original_card_positions: List[position.Position] = [state_card.get_position() for state_card in state.cards]

            agent_positions: List[position.Position] = self.get_visited_positions(
                include_start=include_start_position, start_idx=start_idx)
            reached_card_positions = set(original_card_positions) & set(agent_positions)

            reached_cards: List[card.Card] = list()
            for state_card in state.cards:
                if state_card.get_position() in reached_card_positions:
                    reached_cards.append(state_card)
            return reached_cards
        else:
            return get_changed_cards_along_trajectory(self.get_state_deltas())

    def get_card_scores(self) -> Dict[position.Position, float]:
        # TODO: Partial observability
        scores: Dict[position.Position, float] = dict()
        for initial_card in self.get_initial_cards():
            scores[initial_card.get_position()] = 1. if initial_card in self.get_touched_cards() else 0.
        return scores

    def get_correct_trajectory_distribution(self,
                                            weight_by_time: bool,
                                            full_observability: bool = True,
                                            observation: partial_observation.PartialObservation = None,
                                            observed_positions: Set[position.Position] = None) -> np.array:

        if full_observability and observation:
            raise ValueError('If using full observability, do not need to provide an observation object.')

        distribution: np.array = np.zeros((environment_util.ENVIRONMENT_WIDTH, environment_util.ENVIRONMENT_DEPTH))
        if weight_by_time:
            path: List[position.Position] = [delta.follower.get_position() for delta in self.get_state_deltas()]

            if not full_observability:
                # Remove things from the path that were not observed at this point.
                if not observed_positions:
                    observed_positions = observation.currently_observed_positions()
                new_path: List[position.Position] = list()
                for pos in path:
                    if pos in observed_positions:
                        new_path.append(pos)
                path = new_path

            # The weight is one over the path length, rather than the number of unique locations.
            # The weight is zero if the path is empty (e.g., if the agent is off the path during inference).
            if len(path) == 0:
                weight_per_hex = 0.
            else:
                weight_per_hex: float = 1. / len(path)

            for location in path:
                # Add the weight rather than setting it.
                distribution[location.x][location.y] += weight_per_hex
        else:
            correct_trajectory: List[position.Position] = self.get_visited_positions()

            if not full_observability:
                # Limit by positions visible _right now_. The distribution won't include hexes that aren't currently
                # visible.
                if not observed_positions:
                    observed_positions = observation.currently_observed_positions()
                correct_trajectory = list(set(correct_trajectory) & observed_positions)

            if len(correct_trajectory) == 0:
                weight_per_hex = 0.
            else:
                weight_per_hex: float = 1. / len(correct_trajectory)

            for location in correct_trajectory:
                distribution[location.x][location.y] = weight_per_hex

        return distribution

def construct_game_examples(game: cereal_bar_game.CerealBarGame, max_instruction_index: int):
    # First, segment the actions by instruction.
    all_sets: List[Tuple[int, List[card.Card], List[card.Card]]] = list()

    # Segments actions into instructions
    segmented_actions: List[List[gameplay_action.GameplayAction]] = list(list())
    current_action_sequence: List[gameplay_action.GameplayAction] = list()
    instructions: List[gameplay_action.InstructionAction] = list()

    for action in game.get_actions():
        current_action_sequence.append(action)
        if isinstance(action, gameplay_action.FinishCommandAction):
            segmented_actions.append(current_action_sequence)
            current_action_sequence = list()
        elif isinstance(action, gameplay_action.InstructionAction) and action.completed():
            instructions.append(action)

    assert len(instructions) == len(segmented_actions) or len(instructions) == len(segmented_actions) - 1

    game_examples: List[InstructionExample] = list()
    current_leader_turn_idx: int = 0

    num_steps_remaining: int = 10
    buffer_size: int = 0

    current_observation: partial_observation.PartialObservation = game.get_first_partial_observation()

    for i, segmented_seq in enumerate(segmented_actions):
        follower_action_sequence: List[Tuple[state_delta.StateDelta, agent_actions.AgentAction,
                                             partial_observation.PartialObservation]] = list()
        current_leader_actions_dict: Dict[int, List[gameplay_action.GameplayAction]] = dict()
        current_sets: List[Tuple[List[card.Card], List[card.Card]]] = list()

        following: bool = False
        initial_num_steps_remaining = num_steps_remaining
        initial_buffer_size = buffer_size

        for action in segmented_seq:
            if isinstance(action, gameplay_action.MovementAction):
                if action.get_agent() == environment_objects.ObjectType.FOLLOWER:
                    follower_action_sequence.append((action.get_prior_game_info(), action.get_action(),
                                                     current_observation))

                    if not following:
                        initial_num_steps_remaining = num_steps_remaining
                        initial_buffer_size = buffer_size

                    following = True
                    num_steps_remaining -= 1
                else:
                    if following:
                        if current_leader_turn_idx not in current_leader_actions_dict:
                            current_leader_actions_dict[current_leader_turn_idx] = list()
                        current_leader_actions_dict[current_leader_turn_idx].append(action)

                set_result: Optional[Tuple[List[card.Card], List[card.Card]]] = \
                    state_delta.set_difference(action.get_prior_game_info().cards,
                                               action.get_posterior_game_info().cards)
                if set_result:
                    set_instr_idx = i if following else i - 1
                    all_sets.append((set_instr_idx, set_result[0], set_result[1]))
                    if following:
                        current_sets.append(set_result)

                # Update the observability
                current_observation = partial_observation.update_observation(current_observation,
                                                                             action.get_posterior_game_info())

            elif isinstance(action, gameplay_action.EndTurnAction):
                if action.get_agent() == environment_objects.ObjectType.FOLLOWER:
                    num_steps_remaining = 10
                else:
                    if following and current_leader_turn_idx not in current_leader_actions_dict:
                        current_leader_actions_dict[current_leader_turn_idx] = list()
                    current_leader_turn_idx += 1
            elif isinstance(action, gameplay_action.InstructionAction) and action.completed():
                if following:
                    if current_leader_turn_idx not in current_leader_actions_dict:
                        current_leader_actions_dict[current_leader_turn_idx] = list()
                    current_leader_actions_dict[current_leader_turn_idx].append(action)
                buffer_size += 1
            elif isinstance(action, gameplay_action.FinishCommandAction):
                if not following:
                    initial_buffer_size = buffer_size
                buffer_size -= 1
                follower_action_sequence.append((action.get_prior_game_info(), agent_actions.AgentAction.STOP,
                                                 current_observation))

        leader_actions_to_pass: List[List[gameplay_action.GameplayAction]] = list()
        if current_leader_actions_dict:
            first_leader_turn_idx: int = min(current_leader_actions_dict.keys())
            for j in range(first_leader_turn_idx, max(current_leader_actions_dict.keys()) + 1):
                if j not in current_leader_actions_dict:
                    leader_actions_to_pass.append(list())
                else:
                    leader_actions_to_pass.append((current_leader_actions_dict[j]))
        else:
            first_leader_turn_idx: int = current_leader_turn_idx

        from_game_actions = \
            game.get_leader_actions()[first_leader_turn_idx:first_leader_turn_idx + len(leader_actions_to_pass)]
        assert from_game_actions == leader_actions_to_pass

        example: InstructionExample = \
            InstructionExample(instructions[i].get_tokenized_instruction(),
                               follower_action_sequence,
                               game,
                               i,
                               leader_actions_to_pass,
                               first_leader_turn_idx,
                               current_sets,
                               initial_num_steps_remaining,
                               initial_buffer_size)

        game_examples.append(example)

        if len(game_examples) >= max_instruction_index >= 0:
            break

    return game_examples, all_sets

def construct_examples(games: Dict[str, cereal_bar_game.CerealBarGame],
                       max_instruction_index: int) -> Dict[str, InstructionExample]:
    examples: Dict[str, InstructionExample] = dict()

    with util.get_progressbar('constructing examples', len(games)) as pbar:
        for game_idx, (game_id, game) in enumerate(games.items()):
            game_examples, all_sets = construct_game_examples(game, max_instruction_index)
            game.set_examples(game_examples)
            game.set_sets_made(all_sets)
            for i, example in enumerate(game_examples):
                examples[game_id + '-' + str(i)] = example
            pbar.update(game_idx)

    return examples

def get_example_action_index_pairs(examples: Dict[str, InstructionExample],
                                   full_observability: bool,
                                   observability_refresh_rate: int) -> List[Tuple[str, int]]:
    if not full_observability:
        if observability_refresh_rate > 1:
            raise ValueError('Refreshing after more than one action is not supported.')
        ids = list()
        for example_id, example in examples.items():
            for i in range(len(example.get_action_sequence())):
                ids.append((example_id, i))
        return ids
    return [(key, -1) for key in examples.keys()]


#!/usr/bin/python3
"""1-hbnb"""

from flask import Flask

app = Flask(__name__)
app.url_map.strict_slashes = False

@app.route("/")
def function():
    """function"""
    return "¡Hola HBNB!"

@app.route("/hbnb")
def function2():
    """function2"""
    return "HBNB"

@app.route("/c/<text>")
def function3(text):
    """function2"""
    text = text.replace("_", " ")
    return "C {}".format(text)

@app.route("/python")
@app.route("/python/<text>")
def function4(text):
    """function2"""
    if text is None:
        return "Python is cool"
    text = text.replace("_", " ")
    return "Python {}".format(text)

@app.route("/number/<int:n>")
def function_new(n):
    """function2"""
    return ("{} is a number".format(n))

if __name__ == "__main__":
    app.run(host="0.0.0.0", port=5000, debug=True)