# ---------------------------------------------------------------------------------------------------------- #
# Author: Victor Cardel                                                                                #
# ---------------------------------------------------------------------------------------------------------- #
import tensorflow as tf
import sys
import numpy as np
import cv2
import os

# ---------------------------------------------------------------------------------------------------------- #
# Linha de comando:                                                                                #
#         python3 train_and_test.py /path/to/train_folder /path/to/test_folder /path/to/output.txt           #
# Descrição:                                                                                #
#         * carrega todas as imagens nos subdiretórios de /path/to/train_folder e cria modelo de             #
#           classificação                                                                                #
#         * carrega todas as imagens a serem classificadas no diretório /path/to/test_folder                 #
#         * para cada imagem de teste carregada, imprime uma linha no arquivo /path/to/output.txt com dois   #
#           valores separados por um espaço, sendo eles o nome do arquivo da imagem seguido do nome da sua   #
#           respectiva classe ('futebol', 'golfe', 'sinuca' ou 'tenis')                                      #
# ---------------------------------------------------------------------------------------------------------- #

def calcula_tamanho_dos_diretorios():
	global num_treino,num_teste

	for dir_treino in os.listdir(path):
		num_treino += len(os.listdir(path + '/'+ dir_treino))

	num_teste = len(os.listdir(sys.argv[2]))

def preenche_vetores_de_treino_e_teste():
	global treino,teste,labels_treino,tam_classes,num_classes

	k = 0
	diretorios = os.listdir(path)
	diretorios.sort()
	for classe,dir_treino in enumerate(diretorios):
		lista_imagens_classe = os.listdir(path + '/' + dir_treino)
		tam_classes.append(len(lista_imagens_classe))
		for img in lista_imagens_classe:
			imagem = cv2.imread(path + '/' + dir_treino + '/' + img,cv2.IMREAD_COLOR)
			treino[k] = imagem
			one_hot_encoding = np.zeros([num_classes])
			one_hot_encoding[classe] = 1
			labels_treino[k] = one_hot_encoding
			k = k + 1
	k = 0
	for img in os.listdir(sys.argv[2]):
		imagem = cv2.imread(str(sys.argv[2]) + '/' +  img,cv2.IMREAD_COLOR)
		teste[k] = imagem
		k = k + 1

def constroi_validacao():
	global labels_treino,tam_classes,validacao,treino,labels_validacao,labels_treino,treino_oficial,labels_treino_oficial
	#construindo validacao
	classe_atual = 0
	tam = 0
	classe_de_insercao = 0
	cont = 0
	v = 0
	j = 0
	for k in range(len(treino)):
		indice = np.where(labels_treino[k] == 1)
		if indice[0][0] != classe_atual:
			classe_atual = classe_atual + 1
		if cont < len(tam_classes) and tam > tam_classes[cont] - 1:
			classe_de_insercao = classe_de_insercao + 1
			cont = cont + 1
			tam = 0
		if classe_atual == classe_de_insercao:
			validacao[v] = treino[k]
			labels_validacao[v] = labels_treino[k]
			v = v + 1
			tam = tam + 1
		else:
			treino_oficial[j] = treino[k]
			labels_treino_oficial[j] = labels_treino[k]
			j = j + 1

def processa_base():
	global tam_classes
	preenche_vetores_de_treino_e_teste()
	#aplicando os pesos para cada classe
	tam_classes = map(lambda i: int(i*size_of_validation),tam_classes)
	tam_classes = list(tam_classes)
	constroi_validacao()

###############GLOBALS#################
path = sys.argv[1]
heigth = 64
width = 64
channels = 3
num_filters = 64
size_of_validation = 0.2
num_treino,num_teste = 0,0
num_classes = 4
num_features = heigth*width*channels
batch_size = 32
num_iteracoes_treino = 10000
lr = 0.001
#######################################
calcula_tamanho_dos_diretorios()
treino = np.empty([num_treino,heigth,width,channels], dtype=np.uint8)
teste = np.empty([num_teste,heigth,width,channels], dtype=np.uint8)
labels_treino = np.empty([num_treino,num_classes], dtype = np.uint8)
tam_validacao = int(size_of_validation*num_treino)
tam_treino_oficial = num_treino - tam_validacao
tam_classes = []
validacao = np.empty([tam_validacao,heigth,width,channels], dtype=np.uint8)
labels_validacao = np.empty([tam_validacao,num_classes],dtype=np.uint8)
treino_oficial = np.empty([tam_treino_oficial,heigth,width,channels], dtype=np.uint8)
labels_treino_oficial = np.empty([tam_treino_oficial,num_classes], dtype = np.uint8)
#######################################

processa_base()

#creating vector of labels
labels_treino_escalar = np.empty([tam_treino_oficial], dtype = np.uint8)
labels_val = np.empty([tam_validacao], dtype = np.uint8)

for index,one_hot in enumerate(labels_treino_oficial):
	for classe,value in enumerate(one_hot):
		if value == 1:
			labels_treino_escalar[index] = classe
			break

for index,one_hot in enumerate(labels_validacao):
	for classe,value in enumerate(one_hot):
		if value == 1:
			labels_val[index] = classe
			break

#inicializa seu grafo de operacoes
graph = tf.Graph()
with graph.as_default():
	#imagens, labels e learning rate
	img = tf.compat.v1.placeholder(tf.float32,shape=(None,heigth,width,channels))
	labels = tf.compat.v1.placeholder(tf.int64, shape=(None,))
	learning_rate = tf.compat.v1.placeholder(tf.float32)
	training = tf.compat.v1.placeholder(tf.bool)
	#camadas de convolução
	cv1 = tf.compat.v1.layers.conv2d(img,
				     	num_filters,
				     	(3,3),
				     	(1,1),
				     	padding = 'valid',
				     	activation = tf.nn.relu)

	cv2 = tf.compat.v1.layers.max_pooling2d(cv1,
					       (2,2),
					       (2,2),
					       padding = 'valid')

	cv2 = tf.layers.dropout(cv2,0.15, training = training)

	cv3 = tf.compat.v1.layers.conv2d(cv2,
				     	 num_filters,
				     	 (3,3),
				     	 (1,1),
				     	 padding = 'valid',
				     	 activation = tf.nn.relu)

	cv4 = tf.compat.v1.layers.max_pooling2d(cv3,
					       (2,2),
					       (2,2),
					       padding = 'valid')

	cv4 = tf.layers.dropout(cv4,0.15, training = training)

	cv5 = tf.compat.v1.layers.conv2d(cv4,
				         num_filters,
				         (3,3),
				         (1,1),
				         padding = 'valid',
				         activation = tf.nn.relu)

	cv6 = tf.compat.v1.layers.max_pooling2d(cv5,
					       (2,2),
					       (2,2),
					       padding = 'valid')

	cv6 = tf.layers.dropout(cv6,0.15, training = training)

	cv7 = tf.compat.v1.layers.conv2d(cv6,
				     	 num_filters,
				        (3,3),
				        (1,1),
				        padding = 'valid',
				        activation = tf.nn.relu)

	cv8 = tf.compat.v1.layers.max_pooling2d(cv7,
					       (2,2),
					       (2,2),
					       padding = 'valid')

	cv8 = tf.layers.dropout(cv8,0.15, training = training)

	#Need to reshape output of cnn for classifier
	shape = cv8.shape
	cv8 = tf.reshape(cv8,[-1,shape[1]*shape[2]*shape[3]])
	#classifier
	d1 = tf.compat.v1.layers.dense(cv8, 1024, activation=tf.nn.relu, name='d1')
	d1 = tf.layers.dropout(d1,0.5, training = training)
	d2 = tf.compat.v1.layers.dense(d1, 64, activation=tf.nn.relu, name='d2')
	d2 = tf.layers.dropout(d2,0.2, training = training)

	output = tf.compat.v1.layers.dense(d2, 4, name='output')
	#loss
	loss = tf.compat.v1.nn.sparse_softmax_cross_entropy_with_logits(labels=labels, logits=output)

	train_op = tf.compat.v1.train.AdamOptimizer(learning_rate=learning_rate).minimize(loss)

	correct = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(output, axis=1), labels), dtype=tf.float32))

	print(correct)

def accuracy(session, Xi, yi):
	batch_size = 32
	cont = 0
	for i in range(0, len(Xi), batch_size):
		X_batch = Xi[i:i+batch_size]
		y_batch = yi[i:i+batch_size]
		ret = session.run([correct], feed_dict = {img : X_batch, labels : y_batch, training : False})
		cont += ret[0]
	return 100.0*cont/len(Xi)

with tf.compat.v1.Session(graph = graph) as session:
	#random initialization of the weigths
	session.run(tf.compat.v1.global_variables_initializer())

	for i in range(num_iteracoes_treino):

		indexes = np.random.permutation(len(treino_oficial))[:batch_size]
		imgs_batch = np.take(treino_oficial, indexes, axis=0)
		labels_batch = np.take(labels_treino_escalar, indexes, axis=0)

		ret = session.run([train_op], feed_dict = {img : imgs_batch, labels : labels_batch, learning_rate : lr, training: True})

		if i%100 == 99:
			print("Iteration #%d" % (i))
			print("TRAIN: ACC=%.5f" % (accuracy(session, treino_oficial, labels_treino_escalar)))
			print("VAL: ACC=%.5f\n" % (accuracy(session, validacao, labels_val)))


import logging

from finance.financedb import FinanceDB

logger = logging.getLogger(__name__)

class DBTable:
    def __init__(self, db, table_name, keys, columns, order_by, **kwargs):
        self._db = db
        self._table_name = table_name
        self._keys = keys
        self._columns = columns
        self._order_by = order_by
        # These are columns that are automatically generated by the database
        self._auto_generate_columns = kwargs.get('auto_generate_columns')
        self._where = kwargs.get('where')
        self._key_values = None
        self._update_count = 0
        self._insert_count = 0

    def __iter__(self):
        """Fetch rows from the table."""
        cursor = self.select()

        for row in cursor:
            yield row

    @property
    def update_count(self):
        """The number of records updated in this session."""
        return self._update_count

    @property
    def insert_count(self):
        """The number of records inserted in this session."""
        return self._insert_count

    def select(self, **kwargs):
        """Fetch data from the table and return a cursor.

        Keyword arguments:
        columns -- subset of columns to fetch (default all columns)
        where -- where clause to filter the result
        order_by -- list of column names with optional direction to sort by
        """
        # keys = kwargs.get('keys', self._keys)
        columns = kwargs.get('columns', self._keys + self._columns)
        order_by = kwargs.get('order_by', self._order_by)
        where = kwargs.get('where', self._where)
        t = kwargs.get('t')

        sql = """
            SELECT {}
            FROM {}
            {}
            {}""".format(', '.join(columns),
                         self._table_name,
                         '' if where is None else 'WHERE {}'.format(where),
                         '' if order_by is None else 'ORDER BY {}'.format(', '.join(order_by)))

        c = self._db.conn.cursor()
        if t is None:
            cursor = c.execute(sql)
        else:
            cursor = c.execute(sql, t)

        return cursor

    def check_exists(self, row_dict):
        """Check the supplied row exists in the table based on the primary key."""
        if self._keys is not None and self._key_values is None:
            # build dictionary of unique column values for lookup to determine update or insert
            cursor = self.select(columns=self._keys)
            self._key_values = set([':'.join([str(c) for c in r]) for r in cursor])
            logger.debug('{} unique keys loaded.'.format(len(self._key_values)))
            for i, key in enumerate(self._key_values):
                if i > 3:
                    break
                logger.debug('Key: {}'.format(key))
        key = ':'.join([str(row_dict[c]) for c in self._keys])
        logger.debug('New key: {}'.format(key))

        return False if self._keys is None else key in self._key_values

    def save(self, row_dict):
        """Save the supplied row to the table."""
        c = self._db.conn.cursor()
        if self._keys is not None and self.check_exists(row_dict):
            update_columns = frozenset(self._columns)
            update_keys = frozenset(self._keys)
            if self._auto_generate_columns is not None:
                update_columns -= frozenset(self._auto_generate_columns)
                update_keys -= frozenset(self._auto_generate_columns)

            sql = """\
                UPDATE {}
                SET {}
                WHERE {}""".format(self._table_name,
                                   ', '.join(['{} = ?'.format(c) for c in update_columns if c in row_dict]),
                                   ' AND '.join(['{} = ?'.format(c) for c in update_keys]))
            t = tuple([row_dict.get(c) for c in list([c for c in update_columns if c in row_dict]) + list(update_keys)])
            self._update_count += 1
        else:
            insert_columns = frozenset(self._keys + self._columns)
            if self._auto_generate_columns is not None:
                insert_columns -= frozenset(self._auto_generate_columns)
            sql = """\
                INSERT INTO {} ({})
                VALUES ({})""".format(self._table_name,
                                      ', '.join(insert_columns),
                                      ', '.join(['?'] * len(insert_columns)))
            t = tuple([row_dict.get(c) for c in insert_columns])
            self._insert_count += 1
        # print(sql)
        # print(t)
        c.execute(sql, t)

class Tran(DBTable):
    def __init__(self, db):
        super().__init__(db, 'tran', ('bank', 'account', 'tran_date', 'description', 'amount'),
                         ('sequence', 'balance',
                          'file_name', 'file_sequence', 'calc_balance', 'head', 'source', 'notes'),
                         ('tran_date', 'sequence'),
                         auto_generate_columns=('sequence',))

        self._trans_accounts = None

    def get_accounts_with_transaction(self, row_dict):
        tran_key_columns = ('tran_date', 'description', 'amount')
        bank_account_columns = ('bank', 'account')
        if self._trans_accounts is None:
            # build dictionary of unique column values for lookup to determine update or insert
            cursor = self.select(columns=('bank', 'account', 'tran_date', 'description', 'amount'))
            self._trans_accounts = dict()
            for r in cursor:
                key = ':'.join([str(r[c]) for c in tran_key_columns])
                bank_account = ':'.join([r[c] for c in bank_account_columns])
                if key not in self._trans_accounts:
                    self._trans_accounts[key] = list()
                self._trans_accounts[key].append(bank_account)
            logger.debug('{} unique keys loaded.'.format(len(self._trans_accounts)))
            for i, key in enumerate(self._trans_accounts.keys()):
                if i > 3:
                    break
                logger.debug('Key: {}'.format(key))
        key = ':'.join([str(row_dict[c]) for c in tran_key_columns])
        logger.debug('New key: {}'.format(key))

        return self._trans_accounts[key] if key in self._trans_accounts else list()

def main():
    from finance import logutil
    #    logging.lastResort = None
    _ = logutil.setup_logging()

    db = FinanceDB()
    _ = db.connect('finance.db')

    tran = Tran(db)
    for i, row in enumerate(tran):
        print(dict(row))
        print([x for x in row])
        print(row[0])

        j = dict(row)

        del j['head']
        tran.save(j)
        j = dict(row)
        j['amount'] = 7.77
        tran.save(j)

        if i > 2:
            break
    tran.save(dict(tran_date='2018-02-23',
                   description='Test transaction',
                   bank='Test',
                   account='Test Account',
                   amount=500.25,
                   file_name='test',
                   file_sequence=7))
    db.disconnect()

if __name__ == '__main__':
    main()


#!/bin/env python3

# Doc: https://python-gtk-3-tutorial.readthedocs.io

import gi

gi.require_version("Gtk", "3.0")
from gi.repository import Gtk

class MyWindow(Gtk.Window):
	def __init__(self):
		super().__init__(title="Ciao")

		# Model
		store = Gtk.ListStore(str, float)
		store.append(["Uno", 3.6])
		store.append(["Due", 7.5])

		# Create ListView
		tree = Gtk.TreeView(model=store)
		# Create text renderer (used for both columns)
		textRenderer = Gtk.CellRendererText()
		# Add first column
		column = Gtk.TreeViewColumn("Beer name", textRenderer, text=0, weight=1)
		tree.append_column(column)
		# Add second column
		column = Gtk.TreeViewColumn("Alc %", textRenderer, text=1, weight=1)
		tree.append_column(column)
		self.add(tree)

	def on_button_clicked(self, widget):
		print("Clickato")
		self.label.set_text("Hello world")

win = MyWindow()
win.connect("destroy", Gtk.main_quit)
win.show_all()
Gtk.main()


# -*- coding: utf-8 -*-
"""
Created on Mon Nov 25 19:37:08 2019
    FIFO
@author: 刘瑜等
源代码文件是：U31FIFO.py
"""
import queue          #导入queue模块
s1='我是一只三酷猫!'
FIFO=queue.Queue()
i=len(s1)-1
while i>=0 :
    FIFO.put(s1[i])
    i-=1
while not FIFO.empty():
    print(FIFO.get())


import numpy as np
import re
import pickle
import feedparser

'''

'''

def loadDataSet():
    postList = [['my', 'dog', 'has', 'flea', 'problems', 'help', 'please'],
                ['maybe', 'not', 'take', 'him', 'to', 'dog', 'park', 'stupid'],
                ['my', 'dalmation', 'is', 'so', 'cute', 'I', 'love', 'him'],
                ['stop', 'posting', 'stupid', 'worthless', 'garbage'],
                ['mr', 'licks', 'ate', 'my', 'steak', 'how', 'to', 'stop', 'him'],
                ['quit', 'buying', 'worthless', 'dog', 'food', 'stupid']]

    classVec = [0, 1, 0, 1, 0, 1]   # 1 for abusive, 0 fro not
    return postList, classVec

def createVovabularyList(dataset):
    vocabSet = set([])
    for document in dataset:
        vocabSet = vocabSet | set(document) # | is a union operation

    return list(vocabSet)

def setOfWords2Vec(vocabList, inputSet):
    returnVec = [0]*len(vocabList)
    for word in inputSet:
        if word in vocabList:
            returnVec[vocabList.index(word)] = 1
        else:
            print('the word %s is not in the Vocabulary'%word)

    return returnVec

def bagOfWords2Vec(vocabList, inputSet):
    returnVec = [0]*len(vocabList)
    for word in inputSet:
        if word in vocabList:
            returnVec[vocabList.index(word)] += 1
        # else:
        #     print('the word %s is not in the Vocabulary'%word)

    return returnVec

def trainNB0(trainMatrix, labels):
    number_of_samples = len(trainMatrix)
    number_of_words = len(trainMatrix[0])
    p_Abusive = sum(labels)/number_of_samples

    #防止因为没出现而造成0的现象，连乘之后导致结果为0
    p0_numerator = np.ones(number_of_words)
    p1_numerator = np.ones(number_of_words)
    '''
    为整数，假设有5个training document， 其中一个词每次都出现，则p0_numerator=6为了让分母与分子不同且分母永远比分子大，就让分母从
    2开始加
    '''
    p0_denominator = 2.0
    p1_denominator = 2.0

    for i in range(number_of_samples):
        if labels[i] == 1:
            #为所有1情况下，各个单词occurance各自之和

            p1_numerator += trainMatrix[i]
            #为所有w的occurance之和
            p1_denominator += sum(trainMatrix[i])

        else:
            p0_numerator += trainMatrix[i]

            p0_denominator += sum(trainMatrix[i])
    # 因为log单调递增切此处概率大于零，可以让log(A*B) = logA+logB，变为和的形式p(w1|c)*p(w2|c) = log(p(w1|c))+log(p(w2|c))

    p0_Vec = np.log(p0_numerator/p0_denominator)
    p1_Vec = np.log(p1_numerator/p1_denominator)

    return p0_Vec, p1_Vec, p_Abusive

def classifyNB(testVec, p0, p1, pA):
    p_0 = sum(testVec * p0) + np.log(pA)
    p_1 = sum(testVec * p1) + np.log(pA)

    if p_0>p_1:
        return 0
    else:
        return 1

def testingNB():
    # vocabulary = list()
    # temp = list()
    # for i in range(len(labels)):
    #     if labels[i] == 1:
    #         temp.append(postList[i])
    postList, labels = loadDataSet()
    vocabulary = createVovabularyList(postList)
    trainMatrix = list()

    for i in range(len(postList)):
        trainMatrix.append(setOfWords2Vec(vocabulary, postList[i]))

    p0, p1, pA = trainNB0(trainMatrix, labels)

    testDoc_1 = ['garbage', 'stupid', 'dog']
    testDoc_2 = ['popo', 'tsy', 'cute']

    testVec_1 = setOfWords2Vec(vocabulary, testDoc_1)
    testVec_2 = setOfWords2Vec(vocabulary, testDoc_2)

    print(classifyNB(testVec_1, p0, p1, pA))
    print(classifyNB(testVec_2, p0, p1, pA))

def getTokens(textString):

    textString = textString.strip()
    words = re.findall(r'\w*', textString)
    tokenList = [word.lower() for word in words if len(word)>2]
    # tokenList = list()
    # for word in words:
    #     if len(word)>3:
    #         tokenList.append(word)
    return tokenList

# with open('ham/1.txt', 'r') as file:
#     line = str(file.read())
#     tokens = getTokens(line)
#     print(tokens)

def testingNB_email():
    testSet = []
    trainingLabels = []
    testLabels = []
    trainingMatrix = []
    fullText = []

    for i in range(1, 26):

        with open('ham/%s.txt'%i, 'r') as file:
            line = str(file.read())
        tokens = getTokens(line)
        trainingMatrix.append(tokens)
        fullText.extend(tokens)
        trainingLabels.append(0)

        with open('spam/%s.txt' % i, 'r') as file:
            line = str(file.read())
        tokens = getTokens(line)
        trainingMatrix.append(tokens)
        trainingLabels.append(1)
        fullText.extend(tokens)
    vocabulary = createVovabularyList(trainingMatrix)
    for _ in range(10):
        index = int(np.random.uniform(len(trainingMatrix)+1)-1)

        testSet.append(trainingMatrix[index])
        testLabels.append(trainingLabels[index])
        del(trainingMatrix[index])
        del(trainingLabels[index])

    #now we get the training set and the testing set

    trainingSet = []
    for item in trainingMatrix:
        trainingSet.append(setOfWords2Vec(vocabulary, item))

    p0, p1, pA = trainNB0(np.array(trainingSet), np.array(trainingLabels))
    error = 0
    for i in range(len(testSet)):
        if classifyNB(np.array(setOfWords2Vec(vocabulary, testSet[i])), p0, p1, pA) != testLabels[i]:
            error += 1
            print('error document is: ', testSet[i])
    error_rate = error/len(testSet)
    return error_rate

# error_rate = testingNB_email()
# print('error rate is :', error_rate)

def calcMostFreq(vocabList, fullText):
    import operator
    freqDict = {}
    for token in vocabList:
        freqDict[token] = fullText.count(token)
    sortedFreq = sorted(freqDict.items(), key=operator.itemgetter(1), reverse=True)
    return sortedFreq[:30]

def localWords(feed1, feed0):
    import feedparser

    docList = []
    classList = []
    fullText = []

    minLen = min(len(feed1['entries']), len(feed0['entries']))
    for i in range(minLen):
        wordList = getTokens(feed1['entries'][i]['summary'])
        docList.append(wordList)
        fullText.extend(wordList)
        classList.append(1)
        wordList = getTokens(feed0['entries'][i]['summary'])
        docList.append(wordList)
        fullText.extend(wordList)
        classList.append(0)

    vocabulary = createVovabularyList(docList)
    top30words = calcMostFreq(vocabulary, fullText)
    for word in top30words:
        if word[0] in vocabulary:
            vocabulary.remove(word[0])

    training_index_Set = list(range(2*minLen))   #因为从两个源各去了minLen长度的内容, 此处需要用list把range转化，否则无法删除操作
    test_index_Set = []
    testClasses = []

    for i in range(20):
        index = int(np.random.uniform(0, len(training_index_Set)))
        test_index_Set.append(training_index_Set[index])
        del(training_index_Set[index])

    trainingMatix = []
    trainingClasses = []

    for index in training_index_Set:
        trainingMatix.append(bagOfWords2Vec(vocabulary, docList[index]))
        trainingClasses.append(classList[index])

    p0, p1, pA = trainNB0(trainingMatix, trainingClasses)
    error = 0
    for index in test_index_Set:
        if classList[index] != classifyNB(bagOfWords2Vec(vocabulary, docList[index]), p0, p1, pA):
            error += 1

    error_rate = float(error)/len(test_index_Set)

    print(error_rate)

    return p0, p1, vocabulary

def getTopWord(ny, sf):
    pSF, pNY, vocabulary = localWords(ny, sf)#ny -->feed1, sf -->feed0
    topSF = []
    topNY = []
    for i in range(len(pSF)):
        if pSF[i]>-6:
            topSF.append((vocabulary[i], pSF[i]))
        if pNY[i]>-6:
            topNY.append((vocabulary[i], pNY[i]))

    sortedSF = sorted(topSF, key=lambda item: item[1], reverse=True)
    print('SF*'*10)
    for item in sortedSF[:10]:
        print(item[0])

    print('NY*'*10)
    sortedNY = sorted(topNY, key=lambda item: item[1], reverse=True)
    for item in sortedNY[:10]:
        print(item[0])

ny = feedparser.parse('http://newyork.craigslist.org/stp/index.rss')
sf = feedparser.parse('http://sfbay.craigslist.org/stp/index.rss')
#if we get rid of some stop words, then we can make the error rate lower

getTopWord(ny, sf)
'''
0.45
SF*SF*SF*SF*SF*SF*SF*SF*SF*SF*
butt
married
all
great
from
hard
workout
man
could
face
NY*NY*NY*NY*NY*NY*NY*NY*NY*NY*
home
did
any
there
breasts
wear
boy
love
well
see

'''

#
# ny = feedparser.parse('http://newyork.craigslist.org/stp/index.rss')
# print(ny['entries'][0]['summary'])


def random_word_generator():
    import random
    words_list = ["Python", "Java", "Django", "Ubuntu", "KUbuntu", "Windows", "QWERTY",
                  "Alpha", "Beta", "Gamma", "Toyota", "Banana", "Ergonomics", "Amazon",
                  "India", "Charlie", "Delta", "Echo", "Jupiter", "Neptune", "Uranus"]
    selected_word = random.choice(words_list)
    # print(selected_word)
    return selected_word

# usage
# word_for_user = random_word_generator()
# print(word_for_user)

def dash_printer(selected_word):
    length_of_selected_word = len(selected_word)
    # dash_printing_worker = ""
    for i in range(0, length_of_selected_word):
        dash_printing_worker += " _ "
    return dash_printing_worker

# usage
# sel_w = "Woman" ##from random_word_generator()## line 1-12
# dash_printer(sel_w)


# Importando o módulo Operating System, para manipulação do arquivo TXT
import os

# Gerando a matriz dos assentos.
def gerar_matriz():
    matriz = []
    for rows in range(8):
        contador = 0
        row = []
        for column in range(5):
            if column == 2:
                row.append('  ')
                contador += 1
            else:
                if rows == 0:
                    row.append('  ')
                    contador += 1
                    row[0] = 'MO'
                elif rows == 1:
                    row.append('A'+f'{contador}')
                    contador += 1
                elif rows == 2:
                    row.append('B'+f'{contador}')
                    contador += 1
                elif rows == 3:
                    row.append('C'+f'{contador}')
                    contador += 1
                elif rows == 4:
                    row.append('D'+f'{contador}')
                    contador += 1
                elif rows == 5:
                    row.append('E'+f'{contador}')
                    contador += 1
                elif rows == 6:
                    row.append('F'+f'{contador}')
                    contador += 1
                elif rows == 7:
                    row.append('G'+f'{contador}')
                    contador += 1
                elif rows == 8:
                    row.append('H'+f'{contador}')
                    contador += 1
        matriz.append(row)
    return matriz

# Atribuindo a matriz ao objeto bus
bus = gerar_matriz()

# Lista dos assentos disponíveis
disponiveis = ['A0', 'A1', 'A3', 'A4', 'B0', 'B1', 'B3', 'B4',
               'C0', 'C1', 'C3', 'C4', 'D0', 'D1', 'D3', 'D4',
               'E0', 'E1', 'E3', 'E4', 'F0', 'F1', 'F3', 'F4',
               'G0', 'G1', 'G3', 'G4']

# Lista dos assentos ocupados
ocupados = []

# Função para perguntar se o usuário deseja algo mais, ou se gostaria de sair da aplicação.
def perguntar():
    print("""
Gostaria de solicitar algo mais?
- Digite \033[32m1\033[m para sim
- Digite \033[31m2\033[m para sair da aplicação""")
    resp = input('[1 - Continuar | 2 - Sair]: ').strip().upper()
    while resp not in ['1', '2']:
        print('\033[31mATENÇÃO: Digite apenas "1" para permanecer ou "2" para sair.\033[m')
        resp = input('[1 - Continuar | 2 - Sair]: ').strip().upper()
    if resp == '1':
        print('\n'*25)
        menu()
    elif resp == '2':
        print('\n'*25, f'Volte sempre, {usuario}!')
        print('\n' * 5)
        exit()

# Função para visualizar a matriz dos assentos (apresemtação visual com quebra de linhas)
def booking():
    print('\n' * 25)
    print('\n=', 'RESERVAS DE ASSENTOS', '=')
    for row in bus:
        for num in row:
            if num == 'XX':
                print('\033[41m', '  ', '\033[m', end=' ')
            elif num == 'MO':
                print('\033[44m', num, '\033[m', end=' ')
            elif num == '  ':
                print(num, end=' ')
            else:
                print('\033[42m', num, '\033[m', end=' ')
        print()
    print('=' * 23, '\n')
    print('\033[mLegenda:\n\n', '\033[0;30;42m   ',
          '\033[m: Assentos disponíveis  ',
          '\033[0;30;41m   ', '\033[m: Assentos Indisponíveis')
    return perguntar()

# Função para comprar passagens
def comprar():
    print('\n' * 25)
    print(f"""\033[32mLista de assentos disponíveis:\033[m
{disponiveis[0:10]}
{disponiveis[10:20]}
{disponiveis[20:30]},
            """)
    num = str(input('Digite a numeração do assento desejado: ')).upper().strip()
    while num not in disponiveis:
        print('\033[31mAssento não disponível.\033[m\n')
        print(f"""\033[32mLista de assentos disponíveis:\033[m
{disponiveis[0:10]}
{disponiveis[10:20]}
{disponiveis[20:30]}
            """)
        num = str(input('Digite a numeração de um assento disponível: ')).upper().strip()
    if num in disponiveis:
        ocupados.append(num)
        disponiveis.remove(num)
        ind = num[0]
        coluna = int(num[1])
        linha = 0
        if ind == 'A':
            linha = 1
        elif ind == 'B':
            linha = 2
        elif ind == 'C':
            linha = 3
        elif ind == 'D':
            linha = 4
        elif ind == 'E':
            linha = 5
        elif ind == 'F':
            linha = 6
        elif ind == 'G':
            linha = 7
        bus[linha][coluna] = 'XX'
    print('\n' * 25)
    print(f'\n{usuario}, sua reserva foi efetuada com sucesso!\n'
          f'A numeração do seu assento é: \033[2;32m{num}.\033[m\n')
    return perguntar()

# Função para trocar os assentos que foram escolhidos
def trocar():
    print('\n' * 25)
    if not ocupados:
        print('Nenhum assento ainda foi comprado.')
        perguntar()
    num = str(input('Digite a numeração do assento que deseja trocar: ')).upper().strip()
    while num not in ocupados:
        print('\033[31mO assento informado não existe ou não foi comprado.\033[m\n')
        print(f"""\033[32mLista de assentos já comprados:\033[m
{ocupados[0:30]}
                """)
        num = str(input('Digite a numeração do assento que deseja trocar: ')).upper().strip()
    if num in ocupados:
        disponiveis.append(num)
        ocupados.remove(num)
        ind = num[0]
        ind2 = num[1]
        coluna = int(num[1])
        linha = 0
        if ind == 'A':
            linha = 1
        elif ind == 'B':
            linha = 2
        elif ind == 'C':
            linha = 3
        elif ind == 'D':
            linha = 4
        elif ind == 'E':
            linha = 5
        elif ind == 'F':
            linha = 6
        elif ind == 'G':
            linha = 7
        bus[linha][coluna] = f'{ind + ind2}'
    print('\n' * 25)
    print('Acento devolvido com sucesso! Agora, escolha o seu novo assento.\n')
    comprar()
    return perguntar()

# Função para imprimir a relação de assentos Vendidos x Disponíveis
def imprimir():
    print('\n' * 25)
    relacao = open(os.path.join('RelacaoAssentos.txt'), 'w')
    relacao.write('Relação dos assentos Disponíveis x Ocupados:')
    relacao.write('\n\n\n')
    relacao.write('Assentos disponíveis: ')
    for item in disponiveis:
        relacao.write(item + ' | ')
    relacao.write('\n\n')
    relacao.write('Assentos ocupados: ')
    for item in ocupados:
        relacao.write(item + ' | ')
    relacao.close()
    print('\n' * 25)
    print('Relação no formato .TXT gerada com sucesso!')
    return perguntar()

# Função para aresentar ao usuário o menu de opções
def menu():
    while True:
        print(f"""
Olá, {usuario}! Em que posso te ajudar?

[1] Comprar passagem individual
[2] Trocar assento já escolhido
[3] Verificar assentos vendidos e disponíveis
[4] Imprimir relação Vendidos x Disponíveis
[5] Sair do programa
""")

        esc = input('Digite a opção desejada: ')
        while esc not in ['1', '2', '3', '4', '5']:
            esc = input('\033[31mATENÇÃO: Escolha apenas um número entre 1 e 5: \033[m')
            print('')

        if esc == '1':
            comprar()
        elif esc == '2':
            trocar()
        elif esc == '3':
            booking()
        elif esc == '4':
            imprimir()
        elif esc == '5':
            print('\n' * 25)
            print(f'\nVolte sempre, {usuario}!')
            print('\n' * 5)
            exit()

# Programa principal
print('-=' * 20)
print(f'\033[34m{"Bem vindo(a)!":^40}\033[m')
print('-=' * 20)
usuario = str(input('Qual o seu nome? ')).strip().upper()

menu()


def insomewhere(X, s):
    for item in X:
        if item == s:
            return True
        elif isinstance(item, list):
            if insomewhere(item, s) == True:
                return True
            else:
                return insomewhere(X[1:], s)
    return False
#A = [[1,2],[3,[[[4],5]],6],7,8]
#print(insomewhere(A, 6))
#print(insomewhere(A, 66))

def unnest(lst, X=[]):
    for item in lst:
        if isinstance(item, int):
            X.append(item)
        elif isinstance(item, list):
            unnest(item, X)
        return unnest(lst[1:], X)
    return X
#A=[[1,2],[3,[[[4],5]],6],7,8]
#print(unnest(A))
#print(A)

def print2d(A):
    print('[')
    for i in A:
        print(' '+str(i))
    print(']')
#print2d([i]*5 for i in range(5))

def triangle(x, tri = [], counter = 0):
    if x == 0 :
        return tri
    elif counter == 0:
        tri.append([x])
    else:
        tri.append(tri[counter-1]+[x])
    return triangle(x-1, tri, counter+1)

#print2d(triangle(5))
#print2d(triangle(10))

def table(f, A, x):
    output = []
    for i in x:
        row = [f(j, i) for j in A]
        output.append(row)
    return output
#print2d(table(pow,[1,2,10],range(10,15)))

def nest(item, x):
    if x == 0:
        return item
    else:
        return nest([item], x-1)
#print(nest(53,10))
#print(unnest(nest(53,10)))

A=[[1],[1]]*5
print(A)
A[3][0]=5
print(A)


# -*- coding: utf-8 -*-
"""
Created on Tue Nov 26 17:03:44 2019

@author: Morimoto Lab
"""
import openmdao.api as om
import numpy as np
from model import CTR
import h5py
import time
import scipy.io
from math import cos,sin,radians
from stiffness import Stiffness

t0=time.time()
# Load the path point
f = h5py.File('path.mat','r')
data_pa = f.get('Pa')
data_pb = f.get('Pb')
data_pa = np.array(data_pa[:,49:])
data_pb = np.array(data_pb[:,49:])
# Rotate the path by 180 degrees
Rot = [[-1,0,0],[0,-1,0],[0,0,1]]
Rot = np.array(Rot)
data_pa = np.dot(Rot,data_pa)
# Entry Angle
theta = 45
Rot2 = np.array([[1,0,0],[0,cos(radians(theta)),-sin(radians(theta))],[0,sin(radians(theta)),cos(radians(theta))]])
data_pa = np.dot(Rot2,data_pa)
data_pb = np.dot(Rot2,data_pb)
# Adjust the base position of the concentric tube robot
"""data_pa[0] = data_pa[0] + 50
data_pa[1] = data_pa[1] - 20
data_pa[2] = data_pa[2] - 120"""

# We'll use the component that was defined in the last tutorial

dim = np.shape(data_pa)
N = dim[1]
E=80
# build the model
prob = om.Problem()
indeps = prob.model.add_subsystem('indeps', om.IndepVarComp())

indeps.add_output('length1', np.ones(N)*(-30))
indeps.add_output('length2', np.ones(N)*(-30))
indeps.add_output('length4', np.ones(N)*(-30))
indeps.add_output('kappa2', .05)
indeps.add_output('d1', 1.370)
indeps.add_output('d2', 1.850)
indeps.add_output('d3', 2.007)
indeps.add_output('d4', 2.400)
indeps.add_output('d5', 2.590)
indeps.add_output('d6', 3.048)
indeps.add_output('l22', np.ones(N)*(-60))
indeps.add_output('psi2', np.ones(N))

"""indeps.add_output('length1', np.ones(N)*(-30))
indeps.add_output('length2', np.ones(N)*(-30))
indeps.add_output('length4', np.ones(N)*(-30))
indeps.add_output('kappa2', .04)
indeps.add_output('d1', 1.2)
indeps.add_output('d2', 1.5)
indeps.add_output('d3', 2.1)
indeps.add_output('d4', 2.5)
indeps.add_output('d5', 2.9)
indeps.add_output('d6', 3.1)
indeps.add_output('l22', np.ones(N)*(-60))
indeps.add_output('psi2', np.ones(N))"""

prob.model.add_subsystem('ctr', CTR())
prob.model.add_subsystem('Stiffness',Stiffness())
# define the component whose output will be constrained
prob.model.add_subsystem('const1', om.ExecComp('x = length1*(cos((kappa2*kb2*(l22 - length2))/(kb1 + kb2 + kb3))*cos(psi2)*sin((kappa2*kb2*(length2))/(kb1 + kb2)) + sin((kappa2*kb2*(l22 - length2))/(kb1 + kb2 + kb3))*cos((kappa2*kb2*(length2))/(kb1 + kb2))*cos(psi2)) - (cos(psi2)*(cos((kappa2*kb2*(l22 - length2))/(kb1 + kb2 + kb3)) - 1)*(kb1 + kb2 + kb3))/(kappa2*kb2) - (cos((kappa2*kb2*(l22 - length2))/(kb1 + kb2 + kb3))*cos(psi2)*(cos((kappa2*kb2*(length2))/(kb1 + kb2)) - 1)*(kb1 + kb2))/(kappa2*kb2) + (sin((kappa2*kb2*(l22 - length2))/(kb1 + kb2 + kb3))*cos(psi2)*sin((kappa2*kb2*(length2))/(kb1 + kb2))*(kb1 + kb2))/(kappa2*kb2)-3',x=np.zeros(N),l22=np.zeros(N),length2=np.zeros(N),length1=np.zeros(N),psi2=np.zeros(N)))
prob.model.add_subsystem('const2', om.ExecComp('y = length1*(cos((kappa2*kb2*(l22 - length2))/(kb1 + kb2 + kb3))*sin((kappa2*kb2*(length2))/(kb1 + kb2))*sin(psi2) + sin((kappa2*kb2*(l22 - length2))/(kb1 + kb2 + kb3))*cos((kappa2*kb2*(length2))/(kb1 + kb2))*sin(psi2)) - (sin(psi2)*(cos((kappa2*kb2*(l22 - length2))/(kb1 + kb2 + kb3)) - 1)*(kb1 + kb2 + kb3))/(kappa2*kb2) - (cos((kappa2*kb2*(l22 - length2))/(kb1 + kb2 + kb3))*sin(psi2)*(cos((kappa2*kb2*(length2))/(kb1 + kb2)) - 1)*(kb1 + kb2))/(kappa2*kb2) + (sin((kappa2*kb2*(l22 - length2))/(kb1 + kb2 + kb3))*sin((kappa2*kb2*(length2))/(kb1 + kb2))*sin(psi2)*(kb1 + kb2))/(kappa2*kb2)-20',y=np.zeros(N),l22=np.zeros(N),length2=np.zeros(N),length1=np.zeros(N),psi2=np.zeros(N)))
prob.model.add_subsystem('const3', om.ExecComp('z = length4 + length1*(cos((kappa2*kb2*(l22 - length2))/(kb1 + kb2 + kb3))*cos((kappa2*kb2*(length2))/(kb1 + kb2)) - sin((kappa2*kb2*(l22 - length2))/(kb1 + kb2 + kb3))*sin((kappa2*kb2*(length2))/(kb1 + kb2))) + (sin((kappa2*kb2*(l22 - length2))/(kb1 + kb2 + kb3))*(kb1 + kb2 + kb3))/(kappa2*kb2) + (cos((kappa2*kb2*(l22 - length2))/(kb1 + kb2 + kb3))*sin((kappa2*kb2*(length2))/(kb1 + kb2))*(kb1 + kb2))/(kappa2*kb2) + (sin((kappa2*kb2*(l22 - length2))/(kb1 + kb2 + kb3))*(cos((kappa2*kb2*(length2))/(kb1 + kb2)) - 1)*(kb1 + kb2))/(kappa2*kb2)+120',z=np.zeros(N),l22=np.zeros(N),length4=np.zeros(N),length1=np.zeros(N),length2=np.zeros(N)))
prob.model.add_subsystem('const4', om.ExecComp('g=l22-length2', g=np.zeros(N),l22=np.zeros(N),length2=np.zeros(N)))
prob.model.add_subsystem('const5', om.ExecComp('k1=kb1'))
prob.model.add_subsystem('const6', om.ExecComp('k2=kb2'))
prob.model.add_subsystem('const12', om.ExecComp('k3=kb3'))
"""prob.model.add_subsystem('const5', om.ExecComp('k1=kb2-kb1'))
prob.model.add_subsystem('const6', om.ExecComp('k2=kb3-kb2'))"""
prob.model.add_subsystem('const7', om.ExecComp('t1=d2-d1'))
prob.model.add_subsystem('const8', om.ExecComp('t2=d4-d3'))
prob.model.add_subsystem('const9', om.ExecComp('t3=d6-d5'))
prob.model.add_subsystem('const10', om.ExecComp('t12=d3-d2'))
prob.model.add_subsystem('const11', om.ExecComp('t23=d5-d4'))

# Connect the variables
prob.model.connect('indeps.d1', ['Stiffness.d1','const7.d1'])
prob.model.connect('indeps.d2', ['Stiffness.d2','const10.d2','const7.d2'])
prob.model.connect('indeps.d3', ['Stiffness.d3','const10.d3','const8.d3'])
prob.model.connect('indeps.d4', ['Stiffness.d4','const11.d4','const8.d4'])
prob.model.connect('indeps.d5', ['Stiffness.d5','const11.d5','const9.d5'])
prob.model.connect('indeps.d6', ['Stiffness.d6','const9.d6'])
prob.model.connect('indeps.l22', ['ctr.l22','const1.l22','const2.l22','const3.l22','const4.l22'])
prob.model.connect('indeps.length1', ['ctr.length1','const1.length1','const2.length1','const3.length1'])
prob.model.connect('indeps.length2', ['ctr.length2','const1.length2','const2.length2','const3.length2','const4.length2'])
prob.model.connect('indeps.length4', ['ctr.length4','const3.length4'])
prob.model.connect('indeps.kappa2', ['ctr.kappa2','const1.kappa2','const2.kappa2','const3.kappa2'])
prob.model.connect('Stiffness.kb1', ['ctr.kb1','const1.kb1','const2.kb1','const3.kb1','const5.kb1'])
prob.model.connect('Stiffness.kb2', ['ctr.kb2','const1.kb2','const2.kb2','const3.kb2','const6.kb2'])
prob.model.connect('Stiffness.kb3', ['ctr.kb3','const1.kb3','const2.kb3','const3.kb3','const12.kb3'])
prob.model.connect('indeps.psi2', ['ctr.psi2','const1.psi2','const2.psi2'])

# setup the optimization
prob.driver = om.ScipyOptimizeDriver()
prob.driver.options['optimizer'] = 'SLSQP'
prob.driver.options['maxiter'] = 1500

# Add design variables
prob.model.add_design_var('indeps.length1', lower=-500, upper=0)
prob.model.add_design_var('indeps.length2', lower=-500, upper=0)
prob.model.add_design_var('indeps.length4', lower=-500, upper=0)
prob.model.add_design_var('indeps.kappa2', lower=0.04, upper=.1)
"""prob.model.add_design_var('indeps.kb1', lower=0, upper=60)
prob.model.add_design_var('indeps.kb2', lower=60, upper=100)
prob.model.add_design_var('indeps.kb3', lower=100, upper=200)"""
prob.model.add_design_var('indeps.l22', lower=-500, upper=-15)
prob.model.add_design_var('indeps.psi2', lower=-3.14, upper=3.14)
"""prob.model.add_design_var('indeps.d1', lower=1, upper=3.5)
prob.model.add_design_var('indeps.d2', lower=1,upper=3.5)
prob.model.add_design_var('indeps.d3', lower=1,upper=3.5)
prob.model.add_design_var('indeps.d4', lower=1,upper=3.5)
prob.model.add_design_var('indeps.d5', lower=1,upper=3.5)
prob.model.add_design_var('indeps.d6', lower=1,upper=3.5)"""
prob.model.add_objective('ctr.f')

# Add constraints
prob.model.add_constraint('const1.x', equals=data_pa[0])
prob.model.add_constraint('const2.y', equals=data_pa[1])
prob.model.add_constraint('const3.z', equals=data_pa[2])
prob.model.add_constraint('const4.g', upper=0)
"""prob.model.add_constraint('const5.k1', lower=0,upper=60)
prob.model.add_constraint('const6.k2', lower=60,upper=100)
prob.model.add_constraint('const12.k3', lower=100,upper=250)"""
"""prob.model.add_constraint('const5.k1', lower=0)
prob.model.add_constraint('const6.k2', lower=0)"""
"""prob.model.add_constraint('const7.t1', lower=0.1,upper=1)
prob.model.add_constraint('const8.t2', lower=0.1,upper=1)
prob.model.add_constraint('const9.t3', lower=0.1,upper=1)
prob.model.add_constraint('const10.t12', lower=0.12, upper=0.15)
prob.model.add_constraint('const11.t23', lower=0.12, upper=0.15)"""
#prob.model.add_constraint('const7.l23', lower=0)
prob.setup()
prob.run_driver()

t1=time.time()
# print the optimal results
print('time:', t1-t0)
print('Length1 =',prob['indeps.length1'])
print('Length3 =',prob['indeps.l22']-prob['indeps.length2'])
print('Length2 =',prob['indeps.length2'])
print('Length4 =',prob['indeps.length4'])
print('kappa2 = ',prob['indeps.kappa2'])
print('kb1 = ',prob['Stiffness.kb1'])
print('kb2 = ',prob['Stiffness.kb2'])
print('kb3 = ',prob['Stiffness.kb3'])
print('d1 = ',prob['indeps.d1'])
print('d2 = ',prob['indeps.d2'])
print('d3 = ',prob['indeps.d3'])
print('d4 = ',prob['indeps.d4'])
print('d5 = ',prob['indeps.d5'])
print('d6 = ',prob['indeps.d6'])
print('l22 = ',prob['indeps.l22'])
print('psi2 = ',prob['indeps.psi2'])

# Save joint values and tube parameters to .mat file
mdict = {'Length1':prob['indeps.length1'],'Length2':prob['indeps.length2'],'Length3':prob['indeps.l22']-prob['indeps.length2'],
        'Length4':prob['indeps.length4'],'l22':prob['indeps.l22'],'kb1':prob['Stiffness.kb1'],'kb2':prob['Stiffness.kb2'],
        'kb3':prob['Stiffness.kb3'],'kappa2':prob['indeps.kappa2'],'d1':prob['indeps.d1'],'d2':prob['indeps.d2'],'d3':prob['indeps.d3'],
        'd4':prob['indeps.d4'],'d5':prob['indeps.d5'],'d6':prob['indeps.d6'],'psi2':prob['indeps.psi2']}
# scipy.io.savemat('D:/Desktop/Fred/CTR/CTR optimization/CTR/Inverse_kinematics/jointvalues/jointvalue_distance_001.mat',mdict)
scipy.io.savemat('D:/Desktop/Fred/CTR_tubeselection/CTR_tubeselection/jointvalues/jonitvalue_ob3_test001.mat',mdict)


from random import random

import Tools
import Organism

def create_empty_list_of_lists(size_x, size_y):
    return [[None for i in range(size_x)] for j in range(size_y)]

class Biotope:
    ecosystem = None # Reference to the ecosystem it belongs to
    organismsArray = None # Array that indicates wich organism is in each place
    size_x, size_y = 100, 100

    def __init__(self, size_x, size_y):
        self.size_x = size_x
        self.size_y = size_y
        self.organismsArray = create_empty_list_of_lists(size_x, size_y)

    def set_Ecosystem(self, E):
        self.ecosystem = E
        for org in self.ecosystem.organisms:
            self.add_org(org, org['status']['coordinates'])

    def add_org(self, organism, x = None, y = None):
        if x == None:
            x_ = organism['status']['coordinates']['x']
            y_ = organism['status']['coordinates']['y']
        elif y == None: # in this case both coordinates are packed in the variable x
            if type(x) == dict:
                x_ = x['x']
                y_ = x['y']
            else:
                x_ = x[0]
                y_ = x[1]
        self.organismsArray[x_][y_] = organism

    def move_organism(self, organism, new_place):
        new_x, new_y = new_place
        old_x = organism['status']['coordinates']['x'] # Como debo acceder?
        old_y = organism['status']['coordinates']['y']
        self.organismsArray[old_x][old_y] = None
        self.organismsArray[new_x][new_y] = self

    def delete_org(self, x, y):
        self.organismsArray[x][y] = None

    def delete_org(self, organism):
        (x, y) = organism['status']['coordinates'].values()
        self.organismsArray[x][y] = None

    def __getitem__(self, x, y = None):
        if y == None:
            return self.organismsArray[x[0]][x[1]]
        else:
            return self.organismsArray[x][y]

    def __setitem__(self, organism, x, y = None):
        if y == None:
            self.organismsArray[x[0]][x[1]] = organism
        else:
            self.organismsArray[x][y] = organism

    def Location_is_OK(self, location):
        return Location_is_OK(location.x, location.y)

    def Location_is_OK(self, x, y):
        if (x >= 0) and (y >= 0) and (x < self.size_x) and (y < self.size_y):
            return True
        else:
            return False

    def seek_free_pos(self, attempts = 10):
        # This is used by an organism in order to move to an empty place
        # or to give birth to a new organism in an empty place
        for i in range(attempts):
            x = int(random() * self.size_x)
            y = int(random() * self.size_y)
            if self.Location_is_OK(x, y):
                if (self.organismsArray[x][y] == None):
                    return (x, y)
        return None

    def seek_free_pos_close_to(self, center, radius, attempts = 1):
        # This is used by an organism in order to move to an empty place
        # or to give birth to a new organism in an empty place
        for i in range(attempts):
            x = int(center[0] + Tools.sRandom() * radius + 0.499999)
            y = int(center[1] + Tools.sRandom() * radius + 0.499999)
            if self.Location_is_OK(x, y):
                if (self.organismsArray[x][y] == None):
                    return (x, y)
        return None

    def evolve(self):
        # Climate changes
        pass


#test2 - query the 2 chromadbs!
#https://python.langchain.com/en/latest/modules/agents/agents/custom_multi_action_agent.html
import logging
import os                  #access the chroma folder/files
import re                  #regex to replace multiple spaces
import chromadb            #store data locally for querying

from langchain                  import PromptTemplate
from langchain.embeddings       import HuggingFaceEmbeddings
from langchain.vectorstores     import Chroma
from langchain.chains           import RetrievalQA
from langchain.memory           import ConversationBufferMemory

from langchain.agents           import Tool, AgentExecutor, initialize_agent, AgentType
from langchain.agents           import BaseSingleActionAgent
from langchain.agents           import BaseMultiActionAgent
from langchain.prompts          import StringPromptTemplate
from langchain                  import LLMChain
from typing                     import List, Tuple, Any, Union
from langchain.schema           import AgentAction, AgentFinish

from langchain.tools            import BaseTool

#logging.basicConfig(level=logging.DEBUG)

ABS_PATH    = os.path.dirname(os.path.abspath(__file__))
RUFF_DB_DIR = os.path.join(ABS_PATH, "ruff_db")
IKEA_DB_DIR = os.path.join(ABS_PATH, "ikea_db")

embeddings = HuggingFaceEmbeddings(model_name="all-MiniLM-L6-v2")

#llm_chain = LLMChain(prompt=prompt, llm=llm)
#tool_names = [tool.name for tool in tools]
#
#class CustomOutputParser(AgentOutputParser):
#    def parse(self, llm_output: str) -> Union[AgentAction, AgentFinish]:
#        if \"Final Answer:\" in llm_output:
#            print(\"we reached a Final Answer\\n\")
#            return AgentFinish(
#                return_values={\"output\": llm_output.split(\"Final Answer:\")[-1].strip()},
#                log=llm_output,
#            )
#        print(\"We are still looping through potential answers\\n\")
#        # Parse out the action and action input
##        regex = r\"Action\\s*\\d*\\s*:(.*?)\\nAction\\s*\\d*\\s*Input\\s*\\d*\\s*:[\\s]*(.*)\"
##        match = re.search(regex, llm_output, re.DOTALL)
##        print(match)
##        if not match:
##            raise ValueError(f\"Could not parse LLM output: `{llm_output}`\")
##        action = match.group(1).strip()
##        action_input = match.group(2)
#        # Return the action and action input
##        return AgentAction(tool=action, tool_input=action_input.strip(\" \").strip('\"'), log=llm_output)
#        return AgentAction(tool=llm_output.strip(\" \").strip('\"'), #tool_input=llm_output.strip(\" \").strip('\"'), log=llm_output)
#
#    output_parser = CustomOutputParser()

#CHAT_CONVERSATIONAL_REACT_DESCRIPTION = 'chat-conversational-react-description'
#CHAT_ZERO_SHOT_REACT_DESCRIPTION = 'chat-zero-shot-react-description'
#CONVERSATIONAL_REACT_DESCRIPTION = 'conversational-react-description'
#REACT_DOCSTORE = 'react-docstore'
#SELF_ASK_WITH_SEARCH = 'self-ask-with-search'
#ZERO_SHOT_REACT_DESCRIPTION = 'zero-shot-react-description'
#agent = initialize_agent(tools, llm, agent="zero-shot-react-description", verbose=True)
#agent.run("How much did the economy grow by?")
#agent = LLMSingleActionAgent(llm_chain=llm_chain,output_parser=output_parser,stop=["Observation:"])
#agent = LLMSingleActionAgent(llm_chain=llm_chain,
#                             output_parser=output_parser,
#                             stop=["Observation:"],
#                             allowed_tools=tool_names
#                            )
#agent_executor = AgentExecutor.from_agent_and_tools(agent=agent, tools=tools, verbose=True)
#agent_executor.run("How much did the economy grow by?")
#

def search_ruff(search_input):
    print("\nwithin search_ruff we are searching for:", search_input)

    client_settings = chromadb.config.Settings(
        chroma_db_impl     ="duckdb+parquet",
        persist_directory  =RUFF_DB_DIR,
        anonymized_telemetry=False        #do not allow tracking of data - just incase
    )

    vectorstore = Chroma(
        collection_name    ="ruff_store",
        embedding_function = embeddings,
        client_settings    = client_settings,
        persist_directory  = RUFF_DB_DIR
    )

    docs = vectorstore.similarity_search_with_score(search_input, k=1) #k=1 as we want 1 answer!
    content = docs[0][0]
    my_str = '"'+str(content)+'"'
    tt = my_str.split("'")
    score = docs[0][1]
    return tt[1]

def search_ikea(search_input):
    print("\nwithin search_ikea, we are searching for:", search_input)

    client_settings = chromadb.config.Settings(
        chroma_db_impl     ="duckdb+parquet",
        persist_directory  =IKEA_DB_DIR,
        anonymized_telemetry=False        #do not allow tracking of data - just incase
    )

    vectorstore = Chroma(
        collection_name    ="ikea_store",
        embedding_function = embeddings,
        client_settings    = client_settings,
        persist_directory  = IKEA_DB_DIR
    )

    docs = vectorstore.similarity_search_with_score(search_input, k=1) #k=1 as we want 1 answer!
    content = docs[0][0]
    my_str = '"'+str(content)+'"'
    tt = my_str.split("'")
    score = docs[0][1]
    return tt[1]

def search_janes(query: str) -> str:
    print("\n...and now I am querying JANES data source....\nI'll be right back.\n")
    return "This is the result from the JANES response that we created"

def query_chromadb():
    #if the folder does not exist then stop as we cannot do anything without data
    if not os.path.exists(RUFF_DB_DIR):
        raise Exception(f"{RUFF_DB_DIR} does not exist, nothing can be queried")
    if not os.path.exists(IKEA_DB_DIR):
        raise Exception(f"{IKEA_DB_DIR} does not exist, nothing can be queried")

    query1 = "Why would I use ruff over flake8 and can I purchase it at the IKEA store?"
#    query1 = "What is the weight of a challenger 2 tank?"

    tools = [
        Tool(
            name = "SearchIKEA",
            func=search_ikea,
            description="use for query information about the IKEA FAQ"
        ),
        Tool(
            name = "SearchRuff",
            func=search_ruff,
            description="use for query information about the Python Ruff linter"
        ),
        Tool(
            name = "SearchJanes",
            func=search_janes,
            description="call this to query the JANES database"
        ),
    ]

    print("Initializing EL.VI.S....")
#    memory = ConversationBufferMemory(memory_key="chat_history")
#keep this here incase we need to do something with history later on
    llm = embeddings

    class FakeAgent(BaseMultiActionAgent):
        """Fake Custom Agent."""

        @property
        def input_keys(self):
            return ["input"]

        def plan(
            self, intermediate_steps: List[Tuple[AgentAction, str]], **kwargs: Any
        ) -> Union[AgentAction, AgentFinish]:
            """Given input, decided what to do. If you cannot provide a correct answer do not guess.

            Args:
                intermediate_steps: Steps the LLM has taken to date,
                    along with observations
                **kwargs: User inputs.

            Returns:
                Action specifying what tool to use.
            """
            if len(intermediate_steps) == 0:
                return [
                    AgentAction(tool="SearchIKEA", tool_input=kwargs["input"], log=""),
                    AgentAction(tool="SearchRuff", tool_input=kwargs["input"], log=""),
                    AgentAction(tool="SearchJanes", tool_input=kwargs["input"], log=""),
                ]
            else:
                return AgentFinish(return_values={"output": "\nplan - I have absolutely no idea"}, log="")

        async def aplan(
            self, intermediate_steps: List[Tuple[AgentAction, str]], **kwargs: Any
        ) -> Union[List[AgentAction], AgentFinish]:
            """Given input, decided what to do. If you cannot provide a correct answer do not guess.

            Args:
                intermediate_steps: Steps the LLM has taken to date,
                    along with observations
                **kwargs: User inputs.

            Returns:
                Action specifying what tool to use.
            """
            if len(intermediate_steps) == 0:
                return [
                    AgentAction(tool="SearchIKEA", tool_input=kwargs["input"], log=""),
                    AgentAction(tool="SearchRuff", tool_input=kwargs["input"], log=""),
                    AgentAction(tool="SearchJanes", tool_input=kwargs["input"], log=""),
                ]
            else:
                return AgentFinish(return_values={"output": "\naplan - I have no idea"}, log="")

    agent = FakeAgent()
    agent_executor = AgentExecutor.from_agent_and_tools(agent=agent, tools=tools, verbose=True, max_iterations=1)

#we can call this directly and the output will be shown in the command-line:
#    agent_executor.run(query1)
#or we can contain the responses into a single string and do something with the result
    doc = agent_executor.run(query1)
#    print(type(doc)) #str
    print(doc)
#ideally, we would store each of the individual responses from each tool so that we can have that individual data available, maybe store it as we are gathering it?

def main():
    query_chromadb()

if __name__ == '__main__':
    main()


import printing_functions as pf

itens_pendentes = [
    "caneca","garrafa","bule","prato","brinquedo","relogio","phone case","bandeja","caixa de papelao"
    ]

itens_concluidos = [

]

#chamando as funções do modulo printing_functions:
pf.imprimir_modelos(itens_pendentes,itens_concluidos)

pf.exibir_modelos_concluidos(sorted(itens_concluidos))

print(f"\npendentes: {itens_pendentes}")
print(f"concluídos: {itens_concluidos}")


import logging
import random

from allennlp.data.iterators import BucketIterator
from allennlp.data.vocabulary import Vocabulary
from allennlp.training.trainer import Trainer
import torch

from data_readers.brown import BrownDatasetReader
from data_readers.linzen import LinzenLMDatasetReader
from data_readers.wsj import WSJDatasetReader
from stack_rnn_LM import StackRNNLanguageModel
from simple_rnn_LM import SimpleRNNLanguageModel

def main():
    # reader = BrownDatasetReader()
    # train_dataset = reader.read("data/brown.txt")
    # vocab = Vocabulary.from_instances(train_dataset)
    # dataset_name = "brown"

    reader = WSJDatasetReader(max_num=5000)
    train_dataset = reader.read("data/treebank_3/raw/wsj")
    vocab = Vocabulary.from_instances(train_dataset)
    dataset_name = "wsj"

    # reader = LinzenLMDatasetReader()
    # train_dataset = reader.read("StackNN/data/linzen/rnn_agr_simple/numpred.train")
    # validation_dataset = reader.read("StackNN/data/linzen/rnn_agr_simple/numpred.val")
    # vocab = Vocabulary.from_instances(train_dataset + validation_dataset)
    # dataset_name = "linzenlm"

    swap_push_pop = True

    # if torch.cuda.is_available():
    #     logging.info("Found and using CUDA.")
    #     device = 0
    # else:
    #     logging.info("No CUDA found.")
    #     device = -1

    # For some reason, this code runs MUCH slower on the GPU.
    device = -1

    model = StackRNNLanguageModel(vocab,
                                  rnn_dim=100,
                                  stack_dim=16,
                                  swap_push_pop=swap_push_pop,
                                  device=None if device == -1 else device)

    optimizer = torch.optim.Adam(model.parameters())
    iterator = BucketIterator(batch_size=16,
                              sorting_keys=[("sentence", "num_tokens")])
    iterator.index_with(vocab)

    if device == 0:
        # Best practice to call this after constructing optimizer.
        model.cuda(device)

    trainer = Trainer(model=model,
                      optimizer=optimizer,
                      iterator=iterator,
                      train_dataset=train_dataset,
                      num_epochs=5,
                      # validation_dataset=validation_dataset,
                      # patience=5
                      cuda_device=device)
    trainer.train()

    filename = dataset_name
    if isinstance(model, StackRNNLanguageModel):
        filename = "stack-" + filename
    if swap_push_pop:
        filename += "-swap"

    with open("saved_models/%s.th" % filename, "wb") as fh:
        torch.save(model.state_dict(), fh)
    vocab.save_to_files("saved_models/vocabulary-%s" % dataset_name)

if __name__ == "__main__":
    main()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""Compute j tot as a function of theta for dm_halos project."""

from scipy import integrate
import numpy as np
import matplotlib.pyplot as plt
import h_funcs as h

g = {}
gsfunc, gpfunc, gdfunc, gsomfunc = [0, 0, 0, 0]

def jval():
    """H func computation."""
    h.g_import()
    # normalizations and f_values
    limit = 1000
    yval = np.logspace(-1, np.log10(50), num=20)
    jst = [integrate.quad(lambda x: x * h.hts(x), 0, y, limit=limit)[0] for y in yval]
    js = [integrate.quad(lambda x: x * h.hs(x), 0, y, limit=limit)[0] for y in yval]
    jp = [integrate.quad(lambda x: x * h.hp(x), 0, y, limit=limit)[0] for y in yval]
    jd = [integrate.quad(lambda x: x * h.hd(x), 0, y, limit=limit)[0] for y in yval]
    jsom = [integrate.quad(lambda x: x * h.hsom(x), 0, y, limit=limit)[0] for y in yval]

    with open("j_func_thetas.txt", 'wb') as outfile:
        np.savez(outfile, jst=np.array(jst), js=np.array(js), jp=np.array(
            jp), jd=np.array(jd), jsom=np.array(jsom), theta=np.array(yval))

if __name__ == '__main__':
    # jval()
    with np.load('j_func_thetas.txt', 'rb') as infile:
        theta = infile['theta']
        js = infile['js']
        jp = infile['jp']
        jd = infile['jd']
        jsom = infile['jsom']
        # plt.plot(theta, js)
        # plt.plot(theta, jp)
        # plt.plot(theta, jd)
    h.g_import()
    # yval = np.logspace(-1, 1, num=20)
    # plt.plot(theta, [h.hsom(y) for y in yval])
    # plt.show()
    print([(y, h.hsom(y)) for y in np.logspace(-4, 1, num=50)])
    print(integrate.quad(lambda x: x * h.hsom(x), 0, 0.1, limit=50))
    print(integrate.quad(lambda x: x * h.hsom(x), 0, 1, limit=50))


from pandas import read_csv, DataFrame
from tqdm import tqdm
import numpy as np
import json

# Load our Data From the CSV File in this Directory
raw_df = read_csv('final/orignal-data/bank-additional-full.csv', delimiter=';')
# raw_df = read_csv('final/bank-additional-full.csv', delimiter=';')

# translation dataframe to be populated
translated_df = DataFrame()

# create an enumerated value for each new field we get
translation_field_bank = dict()

# custom months dictionary
translation_field_bank['month'] = {
    'jan': 1,
    'feb': 2,
    'mar': 3,
    'apr': 4,
    'may': 5,
    'jun': 6,
    'jul': 7,
    'aug': 8,
    'sep': 9,
    'oct': 10,
    'nov': 11,
    'dec': 12
}

translation_field_bank['day_of_week'] = {
    'sun': 1,
    'mon': 2,
    'tue': 3,
    'wed': 4,
    'thu': 5,
    'fri': 6,
    'sat': 7
}

# columns where we need to perform "translation"
translation_columns = [
    "job",
    "marital",
    "education",
    "default",
    "housing",
    "loan",
    "contact",
    "month",
    "day_of_week",
    "poutcome",
    "y"
]

binary_dict = {
    "no": 0,
    "yes": 1,
    "unknown": 2
}

print("Translating Data...")

# iterate through each column in the dataframe
for column_name in tqdm(raw_df.columns):

    new_column_values = np.zeros(len(raw_df), dtype=np.float64)

    if column_name == "emp.var.rate":
        print()

    if column_name not in translation_columns:

        # iterate through each row in the column and copy the row value. These are the default values.
        for index, row_value in enumerate(raw_df[column_name]):
            new_column_values[index] = row_value

        translated_df[column_name] = new_column_values
        continue

    elif column_name == "month":

        # iterate through each row and place the enumerated month instead of the string
        for index, row_value in enumerate(raw_df['month']):
            new_column_values[index] = translation_field_bank['month'][row_value]

        translated_df[column_name] = new_column_values
        continue

    elif column_name == "day_of_week":

        # iterate through each row and place the enumerated day instead of the string
        for index, row_value in enumerate(raw_df['day_of_week']):
            new_column_values[index] = translation_field_bank['day_of_week'][row_value]

        translated_df[column_name] = new_column_values
        continue

    # binary assignment
    if raw_df[column_name][0] == "yes" or raw_df[column_name][0] == "no":
        translation_field_bank[column_name] = binary_dict.copy()

    # create a new, unique dictionary that will populate this enumeration object.
    else:
        translation_field_bank[column_name] = dict()

    # enumeration field, starts at zero and increments by one as a uniuqe field is added to our bank.
    enumeration_idx: int = 0

    # iterate through each row in the column
    for index, row_value in enumerate(raw_df[column_name]):

        # if the row value doesn't exist, add it
        if row_value not in translation_field_bank[column_name].keys():
            translation_field_bank[column_name][row_value] = enumeration_idx
            enumeration_idx += 1

        new_column_values[index] = translation_field_bank[column_name][row_value]

    translated_df[column_name] = new_column_values

# rename target data
translated_df = translated_df.rename(columns={'y': 'target'})

# if you want to look at the dataframe
translated_df.to_csv("translated.df.csv")

with open("translation_field_bank.json", "w") as f:
    json.dump(translation_field_bank, f)


import matplotlib.pyplot as plt

class Evaluation:

    @staticmethod
    def write_eval_file(filename, data_dict):
        with open(filename, "w") as f:
            for k, v in data_dict.items():
                f.write(k + "\t[")
                for x in v[:-1]:
                    f.write(x + ", ")
                if len(v) > 0:
                    f.write(v[-1] + "]\n")
                else:
                    f.write("]\n")

    @staticmethod
    def fill_matches_dict_from_file(file_path):
        d = {}
        file = open(file_path).read()
        lines = file.split('\n')
        for l in lines:
            if l !='':
                l_list = l.split('\t')
                matches = [x.strip() for x in l_list[1].replace('[', '').replace(']', '').split(',') if x!='']
                d[l_list[0]] = matches
        return d

    @staticmethod
    def print_stats(a, p, r, f):
        print('accuracy %.2f, precision %.2f, recall %.2f, f1 %.2f' % (a, p, r, f))

    @staticmethod
    def get_eval_metrics(p_dict, tr_dict, sum_values, print_values=False):
        TP, FP, FN, TN = Evaluation.evaluate_predictions(p_dict, tr_dict, sum_values, print_values)
        if print_values:
            print("TP %d, FP %d, FN %d, TN %d" % (TP, FP, FN, TN))
        return Evaluation.calculate_accuracy(TP, FP, FN, TN), \
               Evaluation.calculate_precision(TP, FP), \
               Evaluation.calculate_recall(TP, FN), \
               Evaluation.calculate_f1(TP, FP, FN)

    @staticmethod
    def evaluate_predictions(p_dict, tr_dict, sum_values, print_values):
        TP, FP, FN = Evaluation.count_tp_fp_fn(p_dict, tr_dict, print_values)
        TN = sum_values - TP - FP - FN
        return TP, FP, FN, TN

    @staticmethod
    def calculate_precision(TP, FP):
        return TP / (TP + FP) if (TP + FP) > 0 else 0

    @staticmethod
    def calculate_recall(TP, FN):
        return TP / (TP + FN) if (TP + FN) > 0 else 0

    @staticmethod
    def calculate_accuracy(TP, FP, FN, TN):
        return TP / (TP + FP + FN + TN)

    @staticmethod
    def calculate_f1(TP, FP, FN):
        p = Evaluation.calculate_precision(TP, FP)
        r = Evaluation.calculate_recall(TP, FN)
        return 2 * (p * r) / (p + r) if (p + r) > 0 else 0

    @staticmethod
    def count_tp_fp_fn(p_dict, tr_dict, print_values=False):
        TP = 0
        FP = 0
        FN = 0

        for key, value in p_dict.items():
            for v in value:
                if v in tr_dict[key]:
                    TP += 1
                else:
                    FP += 1
                    if print_values:
                        print("FP(" + key + "): predicted: " + str(v) + ' should be: ' + str(tr_dict[key]))

        for key, value in tr_dict.items():
            if key not in p_dict.keys() and tr_dict[key] != []:
                FN += len([x for x in tr_dict[key] if x != ''])
                if print_values:
                    print("FN(" + key + "): predicted: NONE should be: " + str(tr_dict[key]))
            elif key in p_dict.keys():
                for v in value:
                    if v not in p_dict[key] and v != '':
                        FN += 1
                        if print_values:
                            print("FN(" + key + "): predicted: " + str(p_dict[key]) + ' should contain: ' + str(v))

        return TP, FP, FN

if __name__ == '__main__':

    ########## get baseline ##########
    candidate_path = 'candidates.txt'
    candidate_dict = Evaluation.fill_matches_dict_from_file(candidate_path)
    num_candidates = sum([len(v) for v in candidate_dict.values()])
    print("total number of candidates: %d" %num_candidates)

    ########### get evaluation sets ##########
    annotator1_path = 'eval_set_annotator_1.txt'
    annotator2_path = 'eval_set_annotator_2.txt'

    tr_a1_dict = Evaluation.fill_matches_dict_from_file(annotator1_path)
    tr_a2_dict = Evaluation.fill_matches_dict_from_file(annotator2_path)

    print('number of candidates marked as correct by annotator 1: ' + str(sum([len(v) for v in tr_a1_dict.values()])))
    print('number of candidates marked as correct by annotator 2: ' + str(sum([len(v) for v in tr_a2_dict.values()])))

    tr_union_dict = {}
    tr_intersect_dict = {}

    for key in candidate_dict.keys():
        tr_intersect_dict[key] = list(set(tr_a1_dict[key]).intersection(set(tr_a2_dict[key])))
        tr_union_dict[key] = list(set(tr_a1_dict[key]).union(set(tr_a2_dict[key])))
    Evaluation.write_eval_file("eval_set_union.txt", tr_union_dict)
    Evaluation.write_eval_file("eval_set_intersection.txt", tr_intersect_dict)

    ########## get aggregated evaluation set ##########
    union_path = "eval_set_union.txt"
    intersection_path = "eval_set_intersection.txt"
    tr_union_dict = Evaluation.fill_matches_dict_from_file(union_path)
    tr_intersect_dict = Evaluation.fill_matches_dict_from_file(intersection_path)

    print("number of links in intersection: %d" %sum([len(v) for v in tr_intersect_dict.values()]) )
    print("number of links in union: %d" %sum([len(v) for v in tr_union_dict.values()]) )

    ########## interrater agreement ##########
    interrateragreement, _, _ , _ = Evaluation.get_eval_metrics(tr_a1_dict, tr_a2_dict, num_candidates)
    print('Interrater-agreement:  %.2f' %interrateragreement)

    ########## evaluation of the influence of the threshold ##########
    union_precisions = []
    intersection_precisions = []
    union_recall = []
    intersection_recall = []
    thresholds = []

    for i in [0, 2, 5] + list(range(10,90,10)):
        print('\n')
        print("######################### THRESHOLD 0.%s  #########################" %str(i).zfill(2))
        if i==20:
            print_values = True
        else:
            print_values = False

        pred_path = 'matches_evaluation_th_%s.txt' %str(i).zfill(2)
        pred_dict = Evaluation.fill_matches_dict_from_file(pred_path)

        print("### predictions on union evaluation set: ")
        a,p,r,f = Evaluation.get_eval_metrics(pred_dict, tr_union_dict, num_candidates, print_values)
        Evaluation.print_stats(a,p,r,f)
        union_precisions.append(p)
        union_recall.append(r)

        print("### candidates on union evaluation set: ")
        a,p,r,f = Evaluation.get_eval_metrics(candidate_dict, tr_union_dict, num_candidates)
        Evaluation.print_stats(a,p,r,f)

        print("### predictions on intersection evaluation set: ")
        a,p,r,f = Evaluation.get_eval_metrics(pred_dict, tr_intersect_dict, num_candidates)
        Evaluation.print_stats(a,p,r,f)
        intersection_precisions.append(p)
        intersection_recall.append(r)

        print("### candidates on intersection evaluation set: ")
        a,p,r,f = Evaluation.get_eval_metrics(candidate_dict, tr_intersect_dict, num_candidates)
        Evaluation.print_stats(a,p,r,f)
        thresholds.append(i)

    ########## plotting influence of threshold ##########
    fig1 = plt.plot(intersection_recall, intersection_precisions, 'bo-')
    plt.xlim([0, 1])
    plt.ylim([0, 1])
    for t, x,y in zip(thresholds, intersection_recall, intersection_precisions,):
        label = t
        plt.annotate(label, (x,y), textcoords="offset points")
    plt.show()

    fig2 = plt.plot(thresholds, intersection_precisions, 'bo-')
    plt.xlim([0, 100])
    plt.ylim([0, 1])
    plt.show()

    fig3 = plt.plot(thresholds, intersection_recall, 'bo-')
    plt.xlim([0, 100])
    plt.ylim([0, 1])
    plt.show()

    ########## threshold is at least one common element ##########
    print("######################### THRESHOLD ONE MATCH  #########################")
    pred_path = 'matches_evaluation_one_match.txt'
    pred_dict = Evaluation.fill_matches_dict_from_file(pred_path)
    a, p, r, f = Evaluation.get_eval_metrics(pred_dict, tr_union_dict, num_candidates)
    Evaluation.print_stats(a, p, r, f)


from django.conf.urls import url
from . import views

app_name='customer'

urlpatterns=[
        url(r'^$',views.index,name='index'),
        url(r'^transaction/init/$',views.maketransactions,name='maketransactions'),
        url(r'^viewtransactions/pending/$',views.viewpendingtransactions,name='viewpending'),
        url(r'^transaction/statements/$',views.transactionstatements,name='transactionstatements')
]


# -*- coding: utf-8 -*-
"""
Created on Wed Mar  9 14:12:38 2022

@author: nikel
"""
from tkinter import Tk, StringVar,Y, W, X, PhotoImage, Toplevel, YES, END, Label,ANCHOR, ACTIVE, RIGHT,LEFT, Button, BOTH, TOP, BOTTOM, Frame,  messagebox, filedialog, ttk, Scrollbar, VERTICAL, HORIZONTAL, Checkbutton, Listbox, MULTIPLE
import xlsxwriter
import pandas as pd
from pandas import DataFrame
frame1 = Tk()
frame1.config(bg='white')
frame1.geometry('600x100')
frame1.minsize(width=400, height=50)
frame1.title('AMS CNA')
frame1.columnconfigure(0, weight = 1)
frame1.rowconfigure(1, weight= 1)
frame1.columnconfigure(0, weight = 1)
frame1.rowconfigure(2, weight= 1)
frame1.columnconfigure(0, weight = 1)
frame1.rowconfigure(0, weight= 1)

def selecciondemedida():
    dataws.deiconify()
def cerrardataws():
    dataws.withdraw()

def indicaI():
    indica3['text']='129I'
def indicaU():
    indica3['text']='236U'
def indicaCa():
    indica3['text']='41Ca'
def indicaCl():
    indica3['text']='37Cl'

def abrir_archivo():
    archivo = filedialog.askopenfilename(initialdir ='/',
                                            title='Selecione archivo',
                                            filetype=(('ams files', '*.ams*'),('All files', '*.*')))
    indica['text'] = archivo

def sacardato(lerroa,limiteval):
    indexagarri=[]
    l=0
    for hizki in lerroa:
        if hizki!='NaN' and l>=limiteval:
            indexagarri.append(l)
        l=int(l+1)
    return indexagarri

def Guardarexcel():
    indica2['text']=filedialog.asksaveasfilename(title='Selecione archivo con los números del carrusel', filetype=(('xlsx files', '*.xlsx*'),('All files', '*.*')))
    indica2['text']=indica2['text']+'.xlsx'
    workbook = xlsxwriter.Workbook(indica2['text']) #LocalizaciÃ³n y nombre del Archivo que se quiere obtener xlsx
    worksheet = workbook.add_worksheet()
    datos_obtenidos = indica['text']
    fichero_input= open(datos_obtenidos,"r") #LocalizaciÃ³n y nombre del Archivo que quiere leer
    fa=fichero_input.read()
    p1=fa.find('Blocks')
    p2=fa.find('# Analyses')
    documento=fa[p1+7:p2-1]
    documento=documento.split("\n")
    k=1
    j=0
    limiteval=7
    if indica3['text']=='236U':
        sonnumeros=[0 ,2, 3,7, 8, 9, 10, 11, 12, 13,14, 15, 16, 17,18,19, 20, 21,22,23,24,25,28,29]
        limite=8
    elif indica3['text']=='37Cl':
        sonnumeros=[]
        limite=4
    elif indica3['text']=='41Ca':
        sonnumeros=[0 ,2, 3,7, 8, 9, 10, 11, 12, 13,14, 15, 16, 17, 20, 21]
        limite=3

    #Seleccionar dependiendo de medida
    variables=[]
    recopilacion={}
    for lerro in documento:
        if k==1:
            cabecero=lerro.split("\t")
            for i in range(len(cabecero)):
                variables.append(cabecero[i])
            k=k+1
        else:
            j=j+1
            lerro=lerro.split("\t")
            nombre=lerro[1]
            if j==1:
                archivodatosmin=[None]*len(variables)
                for x in range(limiteval):
                    archivodatosmin[x]=lerro[x]
                index=sacardato(lerro,limiteval)
                for valores in index:
                    archivodatosmin[valores]=lerro[valores]

            elif j==limite:
                index=sacardato(lerro,limiteval)
                for valores in index:
                    archivodatosmin[valores]=lerro[valores]
                if nombre in recopilacion:
                    tt=recopilacion[nombre]
                    tt.append(archivodatosmin)
                    recopilacion[nombre]=tt
                else:
                    tt=[archivodatosmin]
                    recopilacion[nombre]=tt
                j=0
            else:
                index=sacardato(lerro,limiteval)
                for valores in index:
                    archivodatosmin[valores]=lerro[valores]
    row=0
    column=0
    for item in variables :
        worksheet.write(row, column, item)
        column += 1

    for laginizena in recopilacion:
        archivodedatos=recopilacion[laginizena]
        cnt=1
        for lisra in archivodedatos:
            row+=1
            column=0
            for item in lisra:
                if column in sonnumeros:
                    worksheet.write(row, column, float(item))
                elif column==5:
                    worksheet.write(row, column, int(cnt))
                    cnt+=1
                else:
                    worksheet.write(row, column, item)
                column += 1
    workbook.close()
    fichero_input.close()

def crearlista():
    archivo=archivo = filedialog.askopenfilename(initialdir ='/',
                                            title='Selecione archivo',
                                            filetype=(('Excel files', '*.xlsm*'),('Excel files', '*.xlsx*'),('All files', '*.*')))
    df = pd.read_excel(archivo)
    ll=df.values.tolist()
    lekua=0
    izena=0
    brugakoa=0
    antolaketa={}
    ro=0
    co=0
    for zerrenda in ll:
        co=0
        for elem in zerrenda:
            if elem=='Position':
                lekua=co
                brugakoa=ro
            if elem=='Sample ID':
                izena=co
            co+=1
        ro+=1

    ro=0
    co=0
    for zerrenda in ll:
        if ro>=brugakoa:
            aa='" ' + str(zerrenda[izena]) + ' "'
            antolaketa[aa]=zerrenda[lekua]
        ro+=1
    return antolaketa
def anadircabecero():
    indica2['text']=filedialog.asksaveasfilename(filetype=(('xlsx files', '*.xlsx*'),('All files', '*.*')))
    indica2['text']=indica2['text']+'.xlsx'

    workbook = xlsxwriter.Workbook(indica2['text']) #LocalizaciÃ³n y nombre del Archivo que se quiere obtener xlsx
    worksheet = workbook.add_worksheet()
    datos_obtenidos = indica['text']
    fichero_input= open(datos_obtenidos,"r") #LocalizaciÃ³n y nombre del Archivo que quiere leer
    fa=fichero_input.read()
    p1=fa.find('Blocks')
    p2=fa.find('# Analyses')
    documento=fa[p1+7:p2-1]
    documento=documento.split("\n")
    k=1
    j=0
    limiteval=7
    if indica3['text']=='236U':
        sonnumeros=[0 ,2, 3,7, 8, 9, 10, 11, 12, 13,14, 15, 16, 17,18,19, 20, 21,22,23,24,25,28,29]
        limite=8
    elif indica3['text']=='37Cl':
        sonnumeros=[]
        limite=4
    elif indica3['text']=='41Ca':
        sonnumeros=[0 ,2, 3,7, 8, 9, 10, 11, 12, 13,14, 15, 16, 17, 20, 21]
        limite=3

    #Seleccionar dependiendo de medida
    variables=[]
    recopilacion={}
    for lerro in documento:
        if k==1:
            cabecero=lerro.split("\t")
            for i in range(len(cabecero)):
                variables.append(cabecero[i])
            k=k+1
        else:
            j=j+1
            lerro=lerro.split("\t")
            nombre=lerro[1]
            if j==1:
                archivodatosmin=[None]*len(variables)
                for x in range(limiteval):
                    archivodatosmin[x]=lerro[x]
                index=sacardato(lerro,limiteval)
                for valores in index:
                    archivodatosmin[valores]=lerro[valores]

            elif j==limite:
                index=sacardato(lerro,limiteval)
                for valores in index:
                    archivodatosmin[valores]=lerro[valores]
                if nombre in recopilacion:
                    tt=recopilacion[nombre]
                    tt.append(archivodatosmin)
                    recopilacion[nombre]=tt
                else:
                    tt=[archivodatosmin]
                    recopilacion[nombre]=tt
                j=0
            else:
                index=sacardato(lerro,limiteval)
                for valores in index:
                    archivodatosmin[valores]=lerro[valores]
    row=0
    column=0
    adhesion=crearlista()
    for elem in adhesion:
        if elem=='" Sample ID "':
            variables.append(adhesion[elem])
        else:
            if elem in recopilacion:
                gehitua=[]
                listareco=recopilacion[elem]
                for posizioagehitu in listareco:
                    posizioagehitua=posizioagehitu
                    posizioagehitua.append(adhesion[elem])
                    gehitua.append(posizioagehitua)
                recopilacion[elem]=gehitua

    for item in variables :
        worksheet.write(row, column, item)
        column += 1

    for laginizena in recopilacion:
        archivodedatos=recopilacion[laginizena]
        cnt=1
        for lisra in archivodedatos:
            row+=1
            column=0
            for item in lisra:
                if column in sonnumeros:
                    worksheet.write(row, column, float(item))
                elif column==5:
                    worksheet.write(row, column, int(cnt))
                    cnt+=1
                else:
                    worksheet.write(row, column, item)
                column += 1
    workbook.close()
    fichero_input.close()

boton1 = Button(frame1, text= 'Abrir', bg='blue', command= abrir_archivo)
boton1.grid(column = 0, row = 0, sticky='nsew', padx=10, pady=10)

boton2 = Button(frame1, text= 'Seleccionar elemento \n de la medida ', fg='white', bg='black', command= selecciondemedida)
boton2.grid(column = 1, row = 0, sticky='nsew', padx=10, pady=10)
boton4 = Button(frame1, text= 'Guardar en excel \n con la posición\n del carrusel ', fg='white', bg='green2', command= anadircabecero)
boton4.grid(column = 3, row = 0, sticky='nsew', padx=10, pady=10)
boton3 = Button(frame1, text= 'Guardar en Excel', bg='green', command= Guardarexcel)
boton3.grid(column = 2, row = 0, sticky='nsew', padx=10, pady=10)
indica = Label(frame1, fg= 'white', bg='gray26', text= '' , font= ('Arial',10,'bold') )
indica2 = Label(frame1, fg= 'white', bg='gray26', text= '' , font= ('Arial',10,'bold') )
dataws = Toplevel()
dataws.title('Selección de la medida')
dataws.geometry('50x150')
dataws.withdraw()
indica3 = Label(frame1, fg= 'white', bg='gray26', text= '' , font= ('Arial',10,'bold') )
indica3.grid(column=2, row = 1)
Button(dataws, text='\u00B9'+ '\u00B2'+ '\u2079' + 'I',bg='yellow', command=indicaI).pack(side = TOP,anchor=W, fill=X)
Button(dataws, text="\u00B2\u00B3\u2076U",bg='blue', command=indicaU).pack(side = TOP,anchor=W, fill=X)
Button(dataws, text="\u2074\u00B9Ca",bg='DarkOrchid3', command=indicaCa).pack(side = TOP,anchor=W, fill=X)
Button(dataws, text="\u00B3\u2077Cl",bg='orange', command=indicaCl).pack(side = TOP,anchor=W, fill=X)
Button(dataws, text="Cerrar", bg='red',command=cerrardataws).pack(side = BOTTOM,anchor=W, fill=X)
frame1.mainloop()