import numpy as np
import scipy.special
from scipy.special import comb
from collections import namedtuple
import itertools
import copy as cp
import sys
import time
import re
import pandas as pd

motifInstance = namedtuple("motifInstance", "motif positionHash intensities")
subnetwork = namedtuple("subnetwork", "mat reducedmat originalmat rownames colnames rowsum colsum tipo simid cell")
subsubnetwork = namedtuple("subsubnetwork", "mat colnames rowsum colsum tipo simid cell sub_colnames")
type1define = {'lineage_this_cell':3, 'lineage_other_cell':4, 'other_activator':8, 'inhibitor':9, 'inhibitor_lineage_this_cell':10, 'inhibitor_lineage_other_cell':11,
		'lineage_many_this':5, 'lineage_many_other':6,'lineage_all':7, 'terminal_specific_this':12, 'terminal_specific_other':15, 'terminal_2_this':13, 'terminal_2_other':16, 'terminal_all':14, 'tf':1, 'non_tf':2, 'lin':0}
restrict = {0:[type1define['lin'], type1define['non_tf']],
		    1:[type1define['lineage_this_cell'], type1define['terminal_specific_this'], type1define['terminal_all'], type1define['terminal_2_this']],
		    2:[type1define['lineage_this_cell'], type1define['terminal_specific_this'], type1define['terminal_all'], type1define['terminal_2_this']],
		    3:[type1define['tf'], type1define['non_tf']]}

#tuples that contain the functional requirements o be considered for a motif: (type_number, type, inputs_required, outputs_requireds)
functional_definition = [(0, type1define['lin'], 0, 1), (0, type1define['tf'], 1, 1), (0, type1define['non_tf'], 1, 0)]

class Motif(object):
	restrict = restrict
	def __init__(self, subnet = None, type_to_separate=1, simset='', ID=''):
		self.original_instance = subnet.mat
		self.type_to_separate = type_to_separate
		self.types = subnet.tipo[type_to_separate][subnet.colnames]
		self.colsum = subnet.colsum
		self.matched_instances = [subnet]
		self.matches = {simset:1}
		self.comb_index = None
		self.isomorphs= None
		self.permanentId = ID
		self.createIsomorphicInstances()
	#To do some day: check if isomorphic graphs are also symmetric (it is not a problem since all instances will match with the first isomorphic graph of the symmetry group)
	def createIsomorphicInstances(self):
		steps = [(i, j) for i, j in zip(self.types, self.colsum)]
		steps_set = set(steps)
		steps_count = np.array([steps.count(i) for i in steps_set])
		isomorphs = self.original_instance.reshape((self.original_instance.shape[0], self.original_instance.shape[1],1))
		combnames = np.arange(isomorphs.shape[1]).reshape((1, isomorphs.shape[1]))
		if(np.all(steps_count == 1)):
			self.indexcomb = combnames
			self.isomorphs = isomorphs
			self.comb_index	= combnames
		else:
			permuts = [scipy.special.perm(i, i) for i in steps_count]
			z = np.prod(permuts)
			combinations = []
			#isomorphs = np.zeros((original_instance.shape[0], original_instance.shape[1], z))
			isomorphs = isomorphs.repeat(z, axis = 2)
			combnames = combnames.repeat(z, axis = 0)
			for i, j in enumerate(steps_set):
				if(steps_count[i] > 1):
					ind = np.where([k==j[0] and l==j[1] for k, l in zip(self.types, self.colsum)])[0].tolist()
					combinations.append(list(itertools.permutations(ind)))
			aux = combinations[0]
			for cc in range(1, len(combinations)):
				aux = list(itertools.product(aux, combinations[cc]))
				aux = [i+j for i, j in aux]
			indexcomb = np.array(aux)
			assert indexcomb.shape[0] == isomorphs.shape[2], "Isomorphs: index combinations " + str(indexcomb.shape[0]) + ", isomorph matrix " + str(isomorphs.shape[2])
			for cc in range(1, indexcomb.shape[0]):
				isomorphs[:,indexcomb[0, :],cc] = isomorphs[:,indexcomb[cc, :],cc]
				isomorphs[indexcomb[0, :], :,cc] = isomorphs[indexcomb[cc, :], :,cc]
				combnames[cc, indexcomb[0, :]] = combnames[cc, indexcomb[cc, :]]
			self.isomorphs = isomorphs
			self.comb_index	= combnames
	def compare(self, mat, simset='', memorizeSubnet = True):
		if(not np.all(self.colsum == mat.colsum) or not np.all(self.types == mat.tipo[self.type_to_separate][mat.colnames])):
			return False
		else:
			found =  False
			i = 0
			while(not found and i < self.isomorphs.shape[2]):
				found = np.all(mat.mat == self.isomorphs[:,:,i])
				i+=1
			if(not found):
				return False
			else:
				self.addMatch(mat, simset, memorizeSubnet = True)
				return True
	def addMatch(self, mat, simset='',memorizeSubnet = True):
		if(memorizeSubnet):
			self.matched_instances.append(mat)
		nmatches = self.matches.get(simset, 0) + 1
		self.matches[simset] = nmatches

	def setPermanentId(self, pid=""):
		self.permanentId = pid
	def toString(self):
		return 'mat:' + re.sub('[\s+]','',str(self.isomorphs[:,:,0])) + '|types:'+str(self.types) +'|' + str(self.type_to_separate)
	def getNormalizedPositionNamesNaive(self):
		return [str(self.permanentId) +'_t' + str(self.type_to_separate)+ '.' + str(int(self.types[i])) + '_p' + str(i) for i in range(self.types.shape[0])]
	def getNormalizedPositionNames(self):
		naive_id = self.getNormalizedPositionNamesNaive()
		if(self.isomorphs.shape[2]==1):
			return naive_id
		else:
			list_of_equal_sets = []
			#equals = np.zeros((self.isomorphs.shape[2],self.isomorphs.shape[2]))
			for i in range(self.isomorphs.shape[2]):
				for j in range(i):
					if(np.all(self.isomorphs[:,:,i] == self.isomorphs[:,:,j])):
						#equals[i, j] = 1
						dif_in_order = np.where(self.comb_index[i, :] != self.comb_index[j,:])[0]	#this was a bug
						dif_in_order_names = self.comb_index[i, np.where(self.comb_index[i, :] != self.comb_index[j,:])[0] ]
						#equals2 = np.zeros((dif_in_order.shape[0], dif_in_order.shape[0]))
						for k in range(len(dif_in_order)):
							for l in range(k):
								ind_k_in_i = np.where(self.comb_index[i,:] == dif_in_order_names[k])[0]
								ind_l_in_j = np.where(self.comb_index[j,:] == dif_in_order_names[l])[0]
								if(np.all(self.isomorphs[:,ind_k_in_i,i] == self.isomorphs[:,ind_l_in_j,j]) and np.all(self.isomorphs[ind_k_in_i,:,i] == self.isomorphs[ind_l_in_j,:,j])):
									found_set = False
									for s in list_of_equal_sets:
										if(dif_in_order_names[k] in s):
											found_set = True
											s.add(dif_in_order_names[l])
											break
										elif(dif_in_order_names[l] in s):
											found_set = True
											s.add(dif_in_order_names[k])
											break
									if(not found_set):
										list_of_equal_sets.append({dif_in_order_names[k], dif_in_order_names[l]})
			for i in range(len(list_of_equal_sets)):
				s = list_of_equal_sets[i]
				setname = str(self.permanentId) +'_t' + str(self.type_to_separate)+ '.' + str(int(self.types[next(iter(s))])) + '_set' + str(i)
				for j in s:
					naive_id[j] = setname
		return naive_id

	##returns a hash: {simulation_id:{gene_id:[(cell, normalized_position_in_motif),...]}}
	def getNormalizedPositionsOfInstances(self, removeRandom = True):
		normpos = {}
		posnames = self.getNormalizedPositionNames()
		for match in self.matched_instances:
			if (removeRandom and 'random' in match.simid ):
				continue
			simdict = normpos.get(match.simid, {})
			isorder = self.__getIsomorphOrder(match)
			for g in range(match.colnames.shape[0]):
				simdict.setdefault(match.colnames[g], []).append((match.cell, posnames[isorder[g]]))
			normpos[match.simid] = simdict
		return normpos

	def __getIsomorphOrder(self, subnet):
		res = None
		for i in range(self.isomorphs.shape[2]):
			if(np.all(subnet.mat == self.isomorphs[:,:,i])):
				res = self.comb_index[i,:]
				break
		if(res is None):
			raise ValueError("One of the instances did not match the isomorphs!")
		return res
	def resetInstances(self):
		#self.matched_instances = [self.matched_instances[0]]
		self.matched_instances.clear()

class motifContainer(object):
	threshold_bin = 0
	MAX_MOT_SIZE = 10
	def __init__(self, simulSet, motifs = None, type_to_separate = 1, type_to_separate_motifs=3,compress_terminal_features = True, addNewMotifs=True, preprocessedNets=None, motifLastId = None, ignoreSelfReg = False):
		networkInstances = simulSet.simulations
		self.path = simulSet.path
		self.set_name = self.path.split("/")[-1]
		if(self.set_name == ''):
			self.set_name = self.path.split("/")[-2]
		self.type_to_separate = type_to_separate	#only used to join terminal features by group
		self.type_to_separate_motifs = type_to_separate_motifs	#used to constrain nodes that can be occupied by a type in motifs
		self.ignoreSelfReg = ignoreSelfReg
		if(preprocessedNets is None):
			self.networks = [self.preprocessNetwork(n, compress_terminal_features) for n in networkInstances]
		else:
			self.networks = preprocessedNets
		if(motifs is None):
			self.motifs = {}
			for i in range(min(networkInstances[0].genome_size, self.MAX_MOT_SIZE)):
				self.motifs[i] = []
		else:
			self.motifs = motifs
		self.addNewMotifs = addNewMotifs
		self.summaryTables = None
		self.functional_definition = functional_definition
	def preprocessNetwork(self, network, compress_terminal_features = True):
		cellnets = []
		#file = open('testfile.txt','w')
		#file.write("NETWORK ")
		for cell in range(network.ncells):

			#file.write('\n**********************************************************\ncell ' + str(cell) + '\n**********************************************************\n')
			tipo = self.typedefine(cell, network)
			this_cell = np.array(network.regtable[[i for i in network.regtable.columns if (('Cell ' + str(cell)) in i)]])
			gene = np.array(network.regtable.gene)
			#Add rows for genes that may be lacking(due to TF removal in previous steps)
			lacking_genes = np.array([i for i in range(np.max(gene)) if i not in gene])
			gene = np.concatenate((gene, lacking_genes), axis = 0)
			this_cell = np.concatenate((this_cell, np.zeros((lacking_genes.shape[0], this_cell.shape[1]))), axis = 0)
			sort_gene = np.argsort(gene)
			gene = gene[sort_gene].astype(int)
			this_cell = this_cell[sort_gene, :]
			#file.write('tipo: ' + str(tipo) + '\nthis_cell:\n ' + str(this_cell) + '\ngene: ' + str(gene) + '\ncompressing:')
			#average terminal features
			if(compress_terminal_features):
				ind = [i for i, g in enumerate(gene) if  type1define['non_tf'] != tipo[0][g]] #reduced matrix only with TFs in rows
				redmat = this_cell[ind, :]
				redgene = gene[ind]
				i_ter = ind[-1] + 1
				for terminal_type in list(set(tipo[self.type_to_separate][[tt for tt in range(i_ter, len(tipo[0])) if tipo[0][tt] == type1define['non_tf']]])):
					#print(terminal_type)
					#file.write('\t\t terminal_type: ' + str(terminal_type))
					aux = np.mean(this_cell[[i for i in range(this_cell.shape[0]) if  terminal_type == tipo[self.type_to_separate][gene[i]]] ,:], axis = 0).reshape((1, redmat.shape[1]))
					redmat = np.concatenate((redmat, aux), axis = 0)
					redgene = np.concatenate((redgene, gene[np.where(tipo[self.type_to_separate][gene] == terminal_type)][0].reshape(1)), axis = 0)
				#file.write('ind: ' +str( ind) + '\nredmat: \n' + str(redmat) + '\nredgene: ' + str(redgene)+ '\ncompressing:')
			else:
				## NOT IMPLEMENTED (just fill the redgene matrix)
				raise Exception("Sorry! Not implemented with compress_terminal_features  = False")
			#binarize
			mat = np.where(np.abs(redmat)>self.threshold_bin, 1, 0)
			#file.write('binarized mat: \n' + str( mat))
			#Add columns for terminal features
			rowgene = redgene
			colgene = np.concatenate((np.array([i for i in range(mat.shape[1])]), rowgene[tipo[0][rowgene] == type1define['non_tf']]))  #in columns, index == number of gene
			mat = np.concatenate((mat, np.zeros((mat.shape[0], np.where(rowgene[tipo[0][rowgene] == type1define['non_tf']])[0].shape[0]))), axis=1)
			redmat = np.concatenate((redmat, np.zeros((redmat.shape[0], np.where(rowgene[tipo[0][rowgene] == type1define['non_tf']])[0].shape[0]))), axis=1)
			#file.write('Added terminal feature rows:\n\trowgene: ' + str(rowgene) + '\n\tcolgene: ' + str(colgene) +'\n\t matrix:\n ' +str(mat) + '\n\tredmat:\n ' + str(redmat))
			#remove genes that are doing nothing (0s in rows and columns)
			coltoretain = np.sort(np.where(np.sum(mat, axis = 0) > 0)[0])
			rowtoretain = np.sort(np.where(np.sum(mat, axis = 1) > 0)[0])
			toretain = np.sort(np.union1d(coltoretain, rowtoretain))
			redmat = redmat[toretain, :][:, toretain]
			mat = mat[toretain, :][:, toretain]
			rowgene = redgene[toretain]
			colgene = colgene[toretain]
			#file.write('Zeros removed:\n\tcoltoretain: ' + str(coltoretain)+ '\n\trowtoretain: '+str( rowtoretain) +'\n\ttoretain: ' + str(toretain) + '\n\tredmat:\n '+ str(redmat) + '\n\tmat: ' + str(mat) + '\n\trowgene: ' + str(rowgene) +'\n\tcolgene: ' +  str(colgene))
			assert mat.shape[0] == mat.shape[1], 'matrix with different dimension sizes'
			assert np.all(rowgene == colgene), 'different order in matrix rows and columns'
			# Sort by type
			sortorder = np.argsort(tipo[self.type_to_separate_motifs][colgene])
			colgene = colgene[sortorder]
			rowgene = rowgene[sortorder]
			mat = mat[sortorder,:]
			mat = mat[:, sortorder]
			redmat = redmat[sortorder,:]
			redmat = redmat[:, sortorder]
			# Remove self regulation if necessary:
			if(self.ignoreSelfReg):
				for i in range(mat.shape[0]):
					mat[i,i] = 0
					#redmat[i,i] = 0
			#finally, order by number of output connections to break symmetry
			colsum = np.sum(mat, axis=0)
			#sort cols and rows by col
			for tsort in np.sort(np.unique(tipo[self.type_to_separate_motifs][colgene])):
				ind = np.where(tipo[self.type_to_separate_motifs][colgene] == tsort)[0]
				if(ind.shape[0] > 1):
					indsorted = ind[np.argsort(colsum[ind])]
					mat[:, ind] = mat[:, indsorted]
					mat[ind, :] = mat[indsorted, :]
					redmat[:, ind] = redmat[:, indsorted]
					redmat[ind, :] = redmat[indsorted, :]
					colsum[ind] = colsum[indsorted]
					colgene[ind] = colgene[indsorted]
			rowsum = np.sum(mat, axis=1)
			rowgene = colgene
			#file.write('ORDERED:\n\tcolsum: ' + str(colsum) + '\n\trowsum: ' + str(rowsum) +'\n\tredmat: ' + str(redmat) + '\n\tmat: ' + str(mat) + '\n\trowgene: ' + str(rowgene) +'\n\tcolgene: ' + str(colgene))
			#add this cell network
			cellnets.append(subnetwork(mat, redmat, this_cell, rowgene, colgene, rowsum, colsum, tipo, network.name, cell))
		#file.close()
		return cellnets

	def typedefine(self, cell, network):
		t = np.array(network.type3)
		t0 = np.zeros(len(network.type3))
		t1 = np.zeros(len(network.type3))
		t2 = np.zeros(len(network.type3))
		t3 = np.zeros(len(network.type3))
		t4 = np.zeros(len(network.type3)) #will not be used
		t5 = np.zeros(len(network.type3))
		init = np.array(network.getInitialExpr(cell))
		opt = np.array(network.getOptimalExpr())
		dict_of_combos = {}
		first_new_value = max(type1define.values()) + 1
		for i in range(t.shape[0]):
			if('lin' in t[i] and '-1' not in t[i]):
				t3[i] = type1define['tf']
				if('[1]' in t[i] and init[i] > 0):
					t0[i] = type1define['lin']
					t1[i] = type1define['lineage_this_cell']
					t2[i] = type1define['lineage_this_cell']
				elif('[1]' in t[i] and init[i] == 0):
					t0[i] = type1define['tf']
					t1[i] = type1define['other_activator']
					t2[i] = type1define['lineage_other_cell']
				elif('[' + str(network.ncells) + ']' in t[i]):
					t0[i] = type1define['lin']
					t1[i] = type1define['lineage_all']
					t2[i] = type1define['lineage_all']
				elif('[0]' in t[i]):
					t0[i] = type1define['tf']
					t1[i] = type1define['other_activator']
					t2[i] = type1define['other_activator']
				else:
					if(init[i] > 0):
						t0[i] = type1define['lin']
						t1[i] = type1define['lineage_many_this']
						t2[i] = type1define['lineage_many_this']
					else:
						t0[i] = type1define['tf']
						t1[i] = type1define['other_activator']
						t2[i] = type1define['lineage_many_other']
			elif('lin' in t[i] and '-1' in t[i]):
				t3[i] = type1define['tf']
				t0[i] = type1define['tf']
				if(init[i] > 0):
					t1[i] = type1define['inhibitor_lineage_this_cell']
					t2[i] = type1define['inhibitor_lineage_this_cell']
				elif(init[i] == 0):
					t1[i] = type1define['inhibitor']
					t2[i] = type1define['inhibitor']
			else:
				t0[i] = type1define['non_tf']
				t3[i] = type1define['non_tf']
				l = len(np.where(opt[i, :] > 0)[0])
				if(l == 1 and opt[i, cell] > 0):
					t1[i] = type1define['terminal_specific_this']
					t2[i] = type1define['terminal_specific_this']
				elif(l == 1 and opt[i, cell] == 0):
					t1[i] = type1define['terminal_specific_other']
					t2[i] = type1define['terminal_specific_other']
				elif(l > 1 and l < network.ncells and opt[i, cell] > 0):
					t1[i] = type1define['terminal_2_this']
					t2[i] = type1define['terminal_2_this']
					dict_of_combos[i] = ''.join([str(j) for j in np.where(opt[i,:] > 0)[0]])
				elif(l > 1 and  l < network.ncells and opt[i, cell] == 0):
					t1[i] = type1define['terminal_2_other']
					t2[i] = type1define['terminal_2_other']
				elif(l == network.ncells):
					t1[i] = type1define['terminal_all']
					t2[i] = type1define['terminal_all']
		set_of_combos = set(dict_of_combos.values())
		t5 = cp.deepcopy(t1)
		if(len(set_of_combos) > 1):
			for c, cnum in zip(set_of_combos, range(len(set_of_combos))):
				for gnum, cc in zip(dict_of_combos.keys(), dict_of_combos.values()):
					if(c == cc):
						t5[gnum] = first_new_value + cnum
		return (t0, t1, t2, t3, t4, t5)
	def findMotifs(self, starting_n = 0, selectBy=0, functionalFilter = True):
		start_time = time.time()
		sim_cont= 0
		for sim in self.networks:
			cell_cont=0
			for subnet in sim:
				subnet_generator = subNetworkGenerator(subnet, selectBy=selectBy, type_to_separate=self.type_to_separate_motifs, starting_n=starting_n)
				try:
					while(not subnet_generator.complete):
						sub = subnet_generator.next()
						if(functionalFilter):
							if(not self.functionalSubnetwork(sub)):
								continue
						if(sub is None):
							break
						n = sub.mat.shape[0]
						sub_present = False
						for mot in self.motifs[n]:
							sub_present = mot.compare(sub, simset=self.set_name, memorizeSubnet = self.addNewMotifs)
							if(sub_present):
								break
						if(not sub_present and self.addNewMotifs):
							newMotif = Motif(sub, self.type_to_separate_motifs, simset=self.set_name, ID = 's' + str(n) + 'mot' + str(len(self.motifs[n])))
							self.motifs[n].append(newMotif)
				except:
					print("Error while searching for motifs: sim ", sim_cont, " cell ", cell_cont, " ", self.set_name,"\n")
					print(sys.exc_info())
					return subnet_generator

				sim_cont += 1
			cell_cont+=1
		return time.time() - start_time
	def functionalSubnetwork(self, sub, definition = 'def0'):
		isfunctional = True
		g = 0
		while(isfunctional and g<sub.colnames.shape[0]):
			for f in self.functional_definition:
				if(sub.tipo[f[0]][sub.colnames[g]] != f[1]):
					continue
				else:
					if(f[2] > sub.rowsum[g] - sub.mat[g,g] or f[3] > sub.colsum[g] - sub.mat[g,g]):
						isfunctional=False
						break
			g+=1
		return isfunctional
	def __iter__(self):
		for i in self.motifs.keys():
			for j in range(len(self.motifs[i])):
				yield self.motifs[i][j]
	def printMotifSummary(self):
		print("Number of motifs: ", [len(self.motifs[i])for i in self.motifs.keys()])
		for i in self.motifs.keys():
			print("********* ", str(i), "  *********")
			for j in range(len(self.motifs[i])):
				print("___ ", str(j), " _____")
				if(len(self.motifs[i][j].matched_instances) == 0):
					print(self.motifs[i][j].original_instance, "\ntypes: ", self.motifs[i][j].types, '\n')
				else:
					print(self.motifs[i][j].original_instance, "\ntypes: ", self.motifs[i][j].types, "\nnames: ", self.motifs[i][j].matched_instances[0].colnames, "\n", self.motifs[i][j].matched_instances[0].simid, " ", self.motifs[i][j].matched_instances[0].cell, "\n" )
	def writeMotifSummary(self, fname = None, outpath = ''):
		if(fname is None):
			fname = outpath + self.set_name + '_motifSummary.txt'
		else:
			fname = outpath + fname
		file = open(fname, 'w')
		file.write("Type definitions:\n" + str(type1define))
		file.write("\n\nNumber of motifs: " + str([len(self.motifs[i])for i in self.motifs.keys()]))
		for i in self.motifs.keys():
			file.write("\n\n********* " + str(i) + "  *********\n")
			for j in range(len(self.motifs[i])):
				file.write("___ " + str(j) + " _____\n")
				file.write(str(self.motifs[i][j].original_instance) + "\ntypes: " + str(self.motifs[i][j].types) + '\n')
				if(len(self.motifs[i][j].matched_instances) > 0):
					file.write("\nnames: " + str(self.motifs[i][j].matched_instances[0].colnames) +
						"\n" + 'simid: ' + str(self.motifs[i][j].matched_instances[0].simid) +
						" cell: " + str(self.motifs[i][j].matched_instances[0].cell) +
						'\nn_matches: ' + str(self.motifs[i][j].matches) +"\n")
		file.close()
	def makeTables(self, addToName='', min_matches = 40, write=True):
		start_time=time.time()
		try:
			self.summaryTables = {}
			self.__makeMotifTable()
			self.__makeTFTable(min_matches)
			self.__JoinGenesByCell()
			self.__makeCellMotifTable()
			self.__makeType2ByPosition()
			if(write):
				self.__writeOutputTables(addToName, min_matches)
		except ValueError as e:
			print('Value error - Error making tables:\n', e.args)
		except:
			print('Some kind of error making tables')
		return time.time() - start_time
	def __setPermanentIds(self):
		for motsize in self.motifs.keys():
			mot_id = 0
			for mot in self.motifs[motsize]:
				mot.setPermanentId(str(motsize) + '_mot' + str(mot_id))
				mot_id+=1
	def __makeMotifTable(self):
		i = 0
		df = pd.DataFrame(columns=['motif_id', 'motif_str', 'motif_size', 'functional'])
		for j in self.motifs.keys():
			for k in range(len(self.motifs[j])):
				sims  = np.array(list(self.motifs[j][k].matches.keys()))
				newsims = sims[[i for i in range(sims.shape[0]) if not sims[i] in df.columns.tolist()]]
				oldsims = sims[[i for i in range(sims.shape[0]) if sims[i] in df.columns.tolist()]]
				newrow = [self.motifs[j][k].permanentId, self.motifs[j][k].toString(), j, str(np.any([self.functionalSubnetwork(mi) for mi in self.motifs[j][k].matched_instances]))] + [self.motifs[j][k].matches[s] if s in oldsims else 0 for s in df.columns[4:]] + [self.motifs[j][k].matches[s] for s in newsims]
				for s in newsims:
					df[s] = [0 for ii in df.index]
				df.loc[i] = newrow
				i+=1
		try:
			self.summaryTables['motif_table'] = df
			self.summaryTables['motifIdList'] = df.motif_id.tolist()
		except:
			print("self.summaryTables not initialized - data frame not stored, call makeTables method\n")
			return df3
	def __makeTFTable(self, min_matches=40):
		assert 'motif_table' in self.summaryTables.keys(), 'call __makeMotifTable first'
## rows with TF/cell identification
		tuplas = [[re.sub('_$','',net[cell.cell].simid) + '_' + str(genenum),re.sub('s[0-9]+_*' ,'',net[cell.cell].simid),re.sub('_$','',net[cell.cell].simid),cell.cell, genenum] + [tt[genenum] for tt in net[cell.cell].tipo]  for net in self.networks for cell in net for genenum in cell.colnames]
		df  = pd.DataFrame(tuplas, columns =  ['gene_unique_id', 'simset','simid','cell', 'gene'] + ['type' + str(i) for i in range(len(self.networks[0][0].tipo))])
		df = df.set_index(['simid', 'cell', 'gene'], drop=False)
		df['number'] = [i for i in range(len(df.index))]
		#columns with motif id
		#motpos_ids = [posname for motlen in self.motifs.keys() for mot in self.motifs[motlen] for posname in set(mot.getNormalizedPositionNames())] #making a set of NormalizedPositionNames was necessary after making position sets
		motpos_ids = [posname for mot in self for posname in set(mot.getNormalizedPositionNames())] #making a set of NormalizedPositionNames was necessary after making position sets
		mot_ids = {i:j for i, j in zip(motpos_ids, range(len(motpos_ids)))}
		#matrix to store the output
		mat=np.zeros((len(tuplas), len(mot_ids)))
		for j in self.motifs.keys():
			for mot in self.motifs[j]:
				##returns a hash: {simulation_id:{gene_id:[(cell, normalized_position_in_motif),...]}}
				mot_matches = mot.getNormalizedPositionsOfInstances()
				for sid in mot_matches.keys():
					for gid in mot_matches[sid].keys():
						for match in mot_matches[sid][gid]:
							sub = re.sub('_$','',sid)
							mat[df.loc[sub,match[0],gid].number, mot_ids[match[1]]]+=1
		df2 = pd.DataFrame(mat, columns = list(mot_ids.keys()))
		df3 = pd.concat([df.reset_index(drop=True), df2], axis=1)
		try:
			self.summaryTables['tfs_cells_sep'] = df3
			#Make also a table with filtered matches
			if(min_matches > 0):
				motids = self.summaryTables['motif_table']['motif_id'].loc[self.summaryTables['motif_table'][self.set_name]>min_matches]
				preserve_columns = [i for i in df3.columns if i.split('_')[0] in list(motids)]
				self.summaryTables['tfs_cells_sep_thresholded'] = df3[list(df3.columns[:10]) + preserve_columns]
		except KeyError:
			print("self.summaryTables not initialized - data frame not stored, call makeTables method\n")
			return df3
	def __JoinGenesByCell(self,  min_matches=40):
		try:
			df = self.summaryTables['tfs_cells_sep']
		except KeyError:
			print("tfs_cells_sep not calculated - data frame not stored, call makeTables method\n")
		colnames = list(df.columns[10:])
		#since cells are going to be joined, put other values in type4
		newmap = {x:x for x in type1define.values()}
		newmap[type1define['lineage_other_cell']] = type1define['lineage_this_cell']
		newmap[type1define['terminal_2_other']] = type1define['terminal_2_this']
		newmap[type1define['terminal_specific_other']] = type1define['terminal_specific_this']
		df['type4'] = [newmap[i] for i in df.type2]
		groupers= ['gene_unique_id','simset', 'simid', 'gene', 'type3', 'type4']
		df2 = df[groupers + colnames].groupby(groupers).sum()
		df2 = df2.reset_index(drop=False)
		#since df is a reference, remember to remove type4, but store it for later
		self.summaryTables['type4'] = self.summaryTables['tfs_cells_sep']['type4']
		self.summaryTables['tfs_cells_sep'] = self.summaryTables['tfs_cells_sep'].drop('type4', axis=1)
		#df2 = df2.set_index(['gene_unique_id'],drop=False)
		self.summaryTables['tfs_cells_joint'] = df2
		if(min_matches > 0):
			motids = self.summaryTables['motif_table']['motif_id'].loc[self.summaryTables['motif_table'][self.set_name]>min_matches]
			preserve_columns = [i for i in df2.columns if i.split('_')[0] in list(motids)]
			self.summaryTables['tfs_cells_joint_thresholded'] = df2[list(df2.columns[:6]) + preserve_columns]

	def __makeCellMotifTable(self):
		tuplas = [[re.sub('s[0-9]+_*' ,'',net[cell.cell].simid),re.sub('_$','',net[cell.cell].simid),cell.cell] for net in self.networks for cell in net]
		df  = pd.DataFrame(tuplas, columns =  ['simset','simid','cell'])
		df = df.set_index(['simid', 'cell'], drop=False)
		df['number'] = [i for i in range(len(df.index))]
		mot_ids = {m[1].permanentId:m[0] for m in enumerate(self)}

		mat=np.zeros((len(tuplas), len(mot_ids)))
		for m in self:
			for subnet in m.matched_instances:
				mat[df.loc[re.sub('_$','',subnet.simid),subnet.cell].number, mot_ids[m.permanentId]]+=1
		df2 = pd.DataFrame(mat, columns = list(mot_ids.keys()))
		df3 = pd.concat([df.reset_index(drop=True), df2], axis=1)
		try:
			self.summaryTables['cellMotifTable'] = df3
		except:
			print("self.summaryTables not initialized - data frame not stored, call makeTables method\n")
			return df3
	def __makeType2ByPosition(self):
		try:
			tab = self.summaryTables['tfs_cells_sep'].set_index('type2', drop=False)
		except:
			print('No tfs_cells_sep Table or SummaryTables not initialized - call makeTables method')
			return
		types = np.unique(tab.type2)
		typenames = {b:a for a, b in type1define.items()}
		colnames = tab.columns[10:]
		original_mat = tab[colnames]
		mat = np.zeros([len(types), len(colnames)])
		for i in range(len(types)):
			mat[i, :] = np.sum(np.array(tab.loc[types[i]][colnames]), axis = 0)
		mat_sum = np.sum(mat, axis = 1).reshape((len(types),1))
		mat_sum2 = np.sum(mat, axis = 0).reshape((1,len(colnames)))
		mat_norm = mat/mat_sum
		mat_norm2 = mat/mat_sum2
		df0 = pd.DataFrame([(i, typenames[i]) for i in types], columns = ['type2', 'typename'])
		df1 = pd.DataFrame(mat, columns = colnames)
		df2 = pd.DataFrame(mat_norm, columns = colnames)
		df3 = pd.DataFrame(mat_norm2, columns = colnames)
		df1 = pd.concat([df0, df1], axis=1)
		df2 = pd.concat([df0, df2], axis=1)
		df3 = pd.concat([df0, df3], axis=1)
		self.summaryTables['motifsByType2_count'] = df1
		self.summaryTables['motifsByType2_prop_groups'] = df2
		self.summaryTables['motifsByType2_prop_motifs'] = df3

	def __writeOutputTables(self, name='', min_mots=40):
		name = '_'.join([self.set_name, name])
		self.summaryTables['motif_table'].to_csv(name + '_motifSummary.csv')
		self.summaryTables['tfs_cells_sep'].to_csv(name + '_geneByCellNormPosition_sep.csv')
		if(min_mots > 0):
			self.summaryTables['tfs_cells_sep_thresholded'].to_csv(name + '_geneByCellNormPosition_sepThreshold'+ str(min_mots) +'.csv')
			self.summaryTables['tfs_cells_joint_thresholded'].to_csv(name + '_geneNormPosition_sumThreshold'+ str(min_mots) +'.csv')
		self.summaryTables['tfs_cells_joint'].to_csv(name + '_geneNormPosition_sum.csv')
		self.summaryTables['cellMotifTable'].to_csv(name + '_CellMotifTable_summary.csv')
		self.summaryTables['motifsByType2_count'].to_csv(name + '_motifsByType2_count.csv')
		self.summaryTables['motifsByType2_prop_groups'].to_csv(name + '_motifsByType2_prop_groups.csv')
		self.summaryTables['motifsByType2_prop_motifs'].to_csv(name + '_motifsByType2_prop_motifs.csv')
		#self.writeMotifSummary(fname = name+'motifSummary.txt')
	def resetInstances(self):
		for n in self.motifs.keys():
			for m in self.motifs[n]:
				m.resetInstances()
	def saveSelf(self, name = ''):
		import _pickle as pck
		filename = self.set_name + '_'+name+ '_motifContainer.pck'
		with open(filename, 'wb') as output:
 			pickler = pck.Pickler(output, -1)
 			pickler.dump(self)
	@staticmethod
	def readMC(filenamein):
		with open(filenamein, 'rb') as f:
			print('opening ' + filenamein)
			import _pickle as pck
			mc = pck.load(f)
		return mc

#subnetwork definition:
# subnetwork namedtuple("subnetwork", "mat reducedmat originalmat rownames colnames rowsum colsum tipo simid cell")
#subsubnetwork = namedtuple("subsubnetwork", "mat colnames rowsum colsum tipo simid cell subcolnames") #subcolnames will contain indices with respect to the original_mat (eg, 0:7), while colnames correspond to the original colnames (eg., 0, 1, 5, 10).
class subNetworkGenerator(object):
	restrict = restrict
	def __init__(self, net, selectBy = 0,type_to_separate=1, starting_n = 0):
		self.original_net = net
		self.selectBy = selectBy
		self.type_to_separate = type_to_separate
		self.original_mat = net.mat
		self.original_nodes = net.colnames
		self.original_tipo = net.tipo[selectBy][self.original_nodes]
		self.original_tipo_sort = net.tipo[type_to_separate][self.original_nodes]
		self.necessary_types = np.array([i for i in self.restrict[selectBy] if (i in self.original_tipo)])
		self.current_n = self.original_nodes.shape[0] if starting_n ==0 else starting_n
		self.complete = False if self.current_n > len(self.necessary_types) else True
		if(starting_n ==0):
			self.current_combinations = np.array([1 for i in self.original_nodes]).reshape((1, self.original_nodes.shape[0]))
			self.current_combinations_shape = 1
			self.current_row = 0
		else:
			self.current_combinations=np.zeros(10)
			self.current_combinations_shape = 0
			self.current_row = 0
			self.generateCombinations()
	def next(self):
		suitable_combination = None
		while(not self.complete and suitable_combination is None):
			if(self.current_combinations is not None):
				suitable_combination = self.current_combinations[self.current_row, :]
			self.current_row += 1
			if(self.current_row >=  self.current_combinations_shape):
				if(self.current_n <= len(self.necessary_types) or self.current_n <= 2):
					self.complete = True
				else:
					self.current_n-=1
					self.current_row = 0
					self.generateCombinations()
		if(suitable_combination is not None):
			suitable_combination = np.where(suitable_combination == 1)[0]
			suitable_combination = suitable_combination[np.argsort(self.original_tipo_sort[suitable_combination])]
			newmat = self.original_mat[suitable_combination, :][:, suitable_combination]
			newcolsum = np.sum(newmat, axis=0)
			for tsort in np.sort(np.unique(self.original_tipo_sort[suitable_combination])):
				ind = np.where(self.original_tipo_sort[suitable_combination] == tsort)[0]
				if(ind.shape[0] > 1):
					indsorted = ind[np.argsort(newcolsum[ind])]
					suitable_combination[ind] = suitable_combination[indsorted]
			newmat = self.original_mat[suitable_combination, :][:, suitable_combination]
			newcolsum = np.sum(newmat, axis=0)
			newnodes = self.original_nodes[suitable_combination]
			return subsubnetwork(newmat, newnodes, np.sum(newmat, axis=1), newcolsum, self.original_net.tipo, self.original_net.simid, self.original_net.cell, suitable_combination)
		else:
			return None
	def generateCombinations(self):
		combmat = np.zeros((comb(self.original_nodes.shape[0], self.current_n, exact=True), self.original_nodes.shape[0]))
		retain = [True for i in range(combmat.shape[0])]
		it = itertools.combinations([i for i in range(self.original_nodes.shape[0])], self.current_n)
		i = 0
		for combin in it:
			if(not np.all([True if t in self.original_tipo[list(combin)] else False for t in self.necessary_types])):
				retain[i] = False
			else:
				auxmat = self.original_mat[combin, :][:, combin]
				scol = np.where(np.sum(auxmat, axis = 0) > 0)[0]
				srow = np.where(np.sum(auxmat, axis = 1) > 0)[0]
				both = np.union1d(scol, srow)
				if(not len(combin) == both.shape[0]):
					retain[i]=False
				else:
					combmat[i, combin] = 1
			i+=1
		combmat = combmat[retain, :]
		if(combmat.shape[0] > 0):
			self.current_combinations = combmat
			self.current_combinations_shape = combmat.shape[0]
		else:
			self.current_combinations = None
			self.current_combinations_shape = 0
			if(self.current_n <= len(self.necessary_types) or self.current_n <= 2):
				self.complete=True

class randomizedContainer(motifContainer):
	#Takes an object of the class motifContained and a string that is used as an ID of the set
	def __init__(self, container, addName = "random_0", randomizeNodeOutputs = True, randomizeNodeInputs = False, addNewMotifs = False, cpmotifs = False):
		self.path = container.path
		self.set_name = container.set_name + addName
		self.type_to_separate = container.type_to_separate
		self.type_to_separate_motifs = container.type_to_separate_motifs
		self.networks = cp.deepcopy(container.networks)
		self.ignoreSelfReg = container.ignoreSelfReg
		self.randomizeNetworks(randomizeNodeOutputs, randomizeNodeInputs)
		if(cpmotifs):
			self.motifs = cp.deepcopy(container.motifs)
		else:
			self.motifs = container.motifs
		self.addNewMotifs = addNewMotifs
		self.functional_definition = functional_definition
	def randomizeNetworks(self, outputs = True, inputs = False):
		for i in range(len(self.networks)):
			for j in range(len(self.networks[i])):
				if(outputs):
					for k in range(self.networks[i][j].mat.shape[1]):
						mask = np.array([False if l ==k and self.ignoreSelfReg else True for l in range(self.networks[i][j].mat.shape[1])])
						neworder=np.arange(self.networks[i][j].mat.shape[0])[mask]
						np.random.shuffle(neworder)
						self.networks[i][j].mat[mask,k] = self.networks[i][j].mat[neworder, k]
						self.networks[i][j].reducedmat[mask,k] = self.networks[i][j].reducedmat[neworder, k]
				if(inputs):
					for k in range(self.networks[i][j].mat.shape[0]):
						mask = np.array([False if l ==k and self.ignoreSelfReg else True for l in range(self.networks[i][j].mat.shape[0])])
						neworder=np.arange(self.networks[i][j].mat.shape[1])[mask]
						np.random.shuffle(neworder)
						self.networks[i][j].mat[k, mask] = self.networks[i][j].mat[k, neworder]
						self.networks[i][j].reducedmat[k, mask] = self.networks[i][j].reducedmat[k, neworder]
				self.networks[i][j] = subnetwork(self.networks[i][j].mat, self.networks[i][j].reducedmat, self.networks[i][j].originalmat, self.networks[i][j].rownames, self.networks[i][j].colnames, np.sum(self.networks[i][j].mat, axis = 1), np.sum(self.networks[i][j].mat, axis = 0), self.networks[i][j].tipo, self.networks[i][j].simid, self.networks[i][j].cell)

class motifContainerHash(motifContainer):
	def __init__(self, simulSet, motifs = None, type_to_separate = 1,type_to_separate_motifs=3, compress_terminal_features = True, addNewMotifs=True, preprocessedNets=None, motifLastId = None, ignoreSelfReg = False):
		networkInstances = simulSet.simulations
		super().__init__(simulSet, motifs, type_to_separate, type_to_separate_motifs, compress_terminal_features, addNewMotifs, preprocessedNets, ignoreSelfReg = ignoreSelfReg)
		self.isplain = False
		if(motifs is None):
			self.motifs = {}
			self.motif_lastID = {}
			for i in range(min(networkInstances[0].genome_size, self.MAX_MOT_SIZE)):
				self.motifs[i] = {}
				self.motif_lastID[i] = 0
		else:
			assert(motifLastId is not None), "need to provide a dictionary with last IDs for motifs"
			self.motifs = motifs
			self.motif_lastID = motifLastId
	def findMotifs(self, starting_n = 0, selectBy=0, functionalFilter=True):
		assert not self.isplain, "motif hash is plain, use super method"
		start_time = time.time()
		sim_cont= 0
		for sim in self.networks:
			cell_cont=0
			for subnet in sim:
				subnet_generator = subNetworkGenerator(subnet, selectBy=selectBy, type_to_separate=self.type_to_separate_motifs, starting_n=starting_n)
				try:
					while(not subnet_generator.complete):
						sub = subnet_generator.next()
						if(functionalFilter):
							if(not self.functionalSubnetwork(sub)):
								continue
						if(sub is None):
							break
						n = sub.mat.shape[0]
						signature = ','.join(str(sub.colsum))
						sub_present = False
						motsig_list = self.motifs[n].get(signature, [])
						for mot in motsig_list:
							sub_present = mot.compare(sub, simset=self.set_name, memorizeSubnet = self.addNewMotifs)
							if(sub_present):
								break
						if(not sub_present and self.addNewMotifs):
							newMotif = Motif(sub, self.type_to_separate_motifs, simset=self.set_name, ID = 's' + str(n) + 'mot' + str(self.motif_lastID[n]))
							self.motif_lastID[n]+=1
							motsig_list.append(newMotif)
							self.motifs[n][signature] = motsig_list
				except:
					print("Error while searching for motifs: sim ", sim_cont, " cell ", cell_cont, " ", self.set_name,"\n")
					print(sys.exc_info())
					return subnet_generator

				sim_cont += 1
			cell_cont+=1
		return time.time() - start_time
	def writeMotifSummary(self, fname = None, outpath = ''):
		if(self.isplain):
			super().writeMotifSummary(fname, outpath)
		else:
			mots_reserva = self.motifs
			self.motifs = self.getPlainMotifList()
			self.isplain = True
			super().writeMotifSummary(fname, outpath)
			self.motifs = mots_reserva
			self.isplain = False
			file.close()
	def getPlainMotifList(self):
		if(self.isplain):
			return self.motifs
		plainmotifs = {}
		for i in self.motifs.keys():
			plainmotifs[i]=[]
			for k in self.motifs[i].keys():
				plainmotifs[i].extend(self.motifs[i][k])
		return plainmotifs
	def becomePlain(self):
		self.motifs = self.getPlainMotifList()
		self.isplain = True
	def __iter__(self):
		for i in self.motifs.keys():
			if(not self.isplain):
				d = []
				for subkey in self.motifs[i].keys():
					d.extend(self.motifs[i][subkey])
			else:
				d = self.motifs[i]
			for mot in d:
				yield mot
	def makeTables(self, addToName = '', min_matches=40):
		if(self.isplain):
			super().makeTables()
		else:
			mots_reserva = self.motifs
			self.motifs = self.getPlainMotifList()
			self.isplain = True
			t = super().makeTables(addToName, min_matches)
			self.motifs = mots_reserva
			self.isplain = False
		return t
	def __setPermanentIds(self):
		assert self.isplain, "not plain, can't perform action"
		super().__setPermanentIds()
	def resetInstances(self):
		if(self.isplain):
			super().resetInstances()
		else:
			for n in self.motifs.keys():
				for sig in self.motifs[n].keys():
					for m in self.motifs[n][sig]:
						m.resetInstances()

class randomizedContainerHash(motifContainerHash):
	def __init__(self, container, addName = "random_0", randomizeNodeOutputs = True, randomizeNodeInputs = False, addNewMotifs = False, cpmotifs = False):
		self.path = container.path
		self.set_name = container.set_name + addName
		self.type_to_separate = container.type_to_separate
		self.type_to_separate_motifs = container.type_to_separate_motifs
		self.networks = cp.deepcopy(container.networks)
		self.ignoreSelfReg = container.ignoreSelfReg
		self.randomizeNetworks(randomizeNodeOutputs, randomizeNodeInputs)
		if(cpmotifs):
			self.motifs = cp.deepcopy(container.motifs)
		else:
			self.motifs = container.motifs
		self.motif_lastID = container.motif_lastID
		self.isplain = container.isplain
		self.addNewMotifs = addNewMotifs
		self.original_container_threshold_bin = container.threshold_bin
		self.functional_definition = functional_definition
	randomizeNetworks = randomizedContainer.__dict__['randomizeNetworks']

class containerRandomizer():
	def __init__(self, container, randsize = 10, randomizeNodeOutputs = True, randomizeNodeInputs = False, addNewMotifs = False):
		self.set_name = container.set_name + 'random_all'
		self.randsize = randsize
		self.original_container = container
		self.addNewMotifs = addNewMotifs
		if(type(container) == motifContainer):
			self.randomclass = randomizedContainer
		elif(type(container) == motifContainerHash):
			self.randomclass = randomizedContainerHash
		self.randomizedContainers = [self.randomclass(container, addName = "random_"+str(i), randomizeNodeOutputs = randomizeNodeOutputs, randomizeNodeInputs = randomizeNodeInputs, addNewMotifs = addNewMotifs, cpmotifs = False) for i in range(randsize)]
	def findMotifsPar(self, starting_n = 0, selectBy=0, functionalFilter=True):
		import multiprocessing
		random_results = multiprocessing.Queue()
		random_times = multiprocessing.Queue()
		jobs = [multiprocessing.Process(target = self.__findMotifsSingleContainer, args = (rcont, starting_n, selectBy, functionalFilter, random_times, random_results)) for rcont in self.randomizedContainers]
		for j in jobs:
			j.start()
		self.randomizedContainers = [random_results.get() for j in jobs]
		self.mergeMotifCountsToOriginalContainer()
		return [random_times.get() for j in jobs]
	def __findMotifsSingleContainer(self,cont, starting_n, selectBy, functionalFilter, random_times, random_results):
		t=cont.findMotifs(starting_n, selectBy, functionalFilter)
		random_times.put(t)
		random_results.put(cont)
	def findMotifs(self, starting_n = 0, selectBy=0, functionalFilter=True):
		times = []
		for rcont in self.randomizedContainers:
			t= rcont.findMotifs(starting_n, selectBy, functionalFilter)
			times.append(t)
		self.mergeMotifCountsToOriginalContainer()
		return times
	def mergeMotifCountsToOriginalContainer(self):
		omot_hash = {o.permanentId:o for o in self.original_container}
		for rcontainer in self.randomizedContainers:
			for rc in rcontainer:
				try:
					omot_hash[rc.permanentId].matches = {**omot_hash[rc.permanentId].matches, **rc.matches}
				except KeyError:
					omot_hash[rc.permanentId] = rc	#this makes nothing actually

class multiSetMotifFinder:
	def __init__(self, path, conditions, type_to_separate = 1, type_to_separate_motifs=3,compress_terminal_features = True, starting_n = 3, selectBy=0, functionalFilter = True,randsize = 10, randomizeNodeOutputs = True, randomizeNodeInputs = False, addNewMotifs = False, cpmotifs = False, hashSearch = True, randomParallel = True, ignoreSelfReg = False):
		self.paths= {i:''.join((path, i)) for i in conditions}
		self.type_to_separate = type_to_separate	#This affects sorting of nodes in subnetwork; they are subdivided according to type[type_to_separate]. If compress_terminal_features, terminal features are summed
		self.type_to_separate_motifs= type_to_separate_motifs	#this affects comparison between subnetwork and motifs. type[type_to_separate_motifs] must be the same as in Motif for every node in subnetwork
		self.compress_terminal_features = compress_terminal_features # Whether to sum up connexions to terminal features of one type, according to type[type_to_separate]
		self.starting_n = starting_n	#Maximum size of motif
		self.selectBy= selectBy		#When iterating over subnetworks, subnetworks that don't have at least one gene of each kind in restrict[selectBy], are discarded
		self.functionalFilter = functionalFilter #When searching, if True, subnetwork that don't include an input and an output for every TF are discarded (lineage TFs are only required to have outputs)
		self.randsize = randsize	#number of random sets per original set
		self.randomizeNodeOutputs = randomizeNodeOutputs #randomize outputs in random sets or not
		self.randomizeNodeInputs = randomizeNodeInputs	#randomize inputs in random sets or not
		self.addNewMotifs = addNewMotifs		#whether to add new motifs when searching in random networks. If multiprocessing with random, it does not work
		self.cpmotifs = cpmotifs			#when creating random network datasets, whether to use deepcopy to copy motifs or not
		self.randomParallel = randomParallel
		self.ignoreSelfReg = ignoreSelfReg
		self.mc = None
		self.ss = None
		if(hashSearch):
			self.containerClass = motifContainerHash
		else:
			self.containerClass = motifContainer
	def fullMotifAnalysis(self, extraname = '', min_matches=40, save_mc = False, save_adex = False):
		import simulationAnalyzerServer as sana		### remember to check this before sbmitting!
		motifs = None
		last_mot_id = None
		for cond in self.paths.keys():
			cond_time = time.time()
			print(cond, ' started...')
			self.ss= sana.simulationSet(self.paths[cond], cond)
			self.mc = self.containerClass(self.ss, motifs = motifs, type_to_separate = self.type_to_separate, type_to_separate_motifs=self.type_to_separate_motifs,compress_terminal_features = self.compress_terminal_features, addNewMotifs=True, preprocessedNets=None, motifLastId=last_mot_id, ignoreSelfReg = self.ignoreSelfReg)
			print('\t', cond, ' read')
			t = self.mc.findMotifs(self.starting_n, self.selectBy, self.functionalFilter)
			print('\t',cond, ' motifs found in ', str(t))
			rc = containerRandomizer(self.mc, randsize=self.randsize, randomizeNodeOutputs = self.randomizeNodeOutputs, randomizeNodeInputs = self.randomizeNodeInputs, addNewMotifs = self.addNewMotifs)
			print('\t',cond, ' random sets generated')
			if(self.randomParallel and not self.addNewMotifs):
				t=rc.findMotifsPar(self.starting_n, self.selectBy, self.functionalFilter)
			else:
				if(self.addNewMotifs):
					print(cond, " Warning!: Not using multiprocessing because addNewMotifs == True")
				t=rc.findMotifs(self.starting_n, self.selectBy, self.functionalFilter)
			print('\t',cond, ' random sets analyzed in ', str(t))
			try:
				t = self.mc.makeTables(extraname, min_matches)
				print('\t',cond, ' tables made in ', str(t))
			except:
				print("Some error making tables")
			finally:
				if(save_mc):
					self.mc.saveSelf(extraname)
					print('\t',cond, ' saved')
			adex = auxiliaryDataExtractor(self.mc, self.ss)
			t = adex.makeTables(extraname)
			if(save_adex):
					adex.saveSelf(extraname)
			print('\t',cond, ' auxiliary tables made in ', str(t))
			self.mc.resetInstances()
			print('\t', cond, ' motifs resetted\n')
			motifs = self.mc.motifs
			if(self.containerClass is motifContainerHash):
				last_mot_id = self.mc.motif_lastID
			print('condition ', cond, 'finished in: ', time.time() - cond_time, '\n***\n')

class auxiliaryDataExtractor():
	def __init__(self, net_container, simul_set):
		self.netcontainer = net_container
		self.simset = simul_set
		self.stored_results = {}
	def makeTables(self, extraname = ''):
		start_time = time.time()
		self.makeTabSkeleton()
		self.getExpressionBySim()
		#print('a')
		self.getMutantPhenotypeByCell('tfs')
		#print('b')
		self.getMutantPhenotypeByCell('sites')
		#print('c')
		self.getNumberOfRegulatorsByGeneByCell()
		#print('d')
		self.getMeanPhenotypeByGeneByCell('tfs')
		#print('e')
		self.getMeanPhenotypeByGeneByCell('sites')
		#print('f')
		self.getCorrelationBetweenTerminalTypes()
		#print('g')
		self.getOverlapOfRegulationSets()
		#print('h')
		self.writeTables(extraname)
		#print('i')
		return time.time() - start_time
	def makeTabSkeleton(self):
		groupers= ['gene_unique_id','simset', 'simid', 'gene', 'type3'] #'type4' removed
		try:
			self.stored_results['skeleton'] = self.netcontainer.summaryTables['tfs_cells_sep'][groupers]
			self.stored_results['skeleton']= self.stored_results['skeleton'].assign(type4=list(self.netcontainer.summaryTables['type4']))
		except KeyError:
			print('makeTabSkeleton KeyError - No tfs_cells_sep')
	def getExpressionBySim(self):
		# Select only TF genes
		df = self.stored_results['skeleton'].loc[self.stored_results['skeleton']['type3'] == type1define['tf']]
		df = df.drop_duplicates()	#in skeleton table there is a row per gene per cell; now only want a row per cell
		#mat = np.zeros([self.stored_results['skeleton'].shape[0], self.simset.simulations[0].ncells]) # store results
		mat = np.zeros([df.shape[0], self.simset.simulations[0].ncells])
		for sim in self.simset.simulations:
			df_index = [i for i,j in enumerate(df.simid) if j == re.sub('_$' ,'',sim.name)]
			mat[df_index, :] = sim.expr[[i for i in sim.expr.columns if 'exp' in i]].loc[list(df['gene'].iloc[df_index])]
		mat = pd.DataFrame(mat, columns = [i for i in sim.expr.columns if 'exp' in i])
		mat_sorted = np.array(mat)
		mat_sorted.sort(axis=1)
		mat_sorted = pd.DataFrame(mat_sorted[:,::-1], columns = ['sorted'+i for i in sim.expr.columns if 'exp' in i])
		self.stored_results['expression_uniqueGeneIds'] = pd.concat([df.reset_index(drop=True), mat, mat_sorted], axis = 1)
	def getMutantPhenotypeByCell(self, mutation_type = 'tfs'):
		df = self.stored_results['skeleton'].loc[self.stored_results['skeleton']['type3'] == type1define['tf']]
		df = df.drop_duplicates()
		typeset = sorted(set(self.simset.simulations[0].type3))
		mat = np.zeros([df.shape[0], self.simset.simulations[0].genome_size*self.simset.simulations[0].ncells])
		t3 = self.simset.simulations[0].type3 #outer name (more general and invariant between cells)
		t = self.simset.simulations[0].type #inner name
		mat_colnames = ['_'.join(['cell'+str(c), t3[g], t[g], str(g)]) for tt in typeset for c in range(self.simset.simulations[0].ncells) for g in range(self.simset.simulations[0].genome_size)  if t3[g]==tt]
		for sim in self.simset.simulations:
			simphen =np.concatenate([sim.getSortedMutEffectsByMean(type3 = [i], mutation_type = mutation_type)[0] for i in typeset], axis=1)
			df_index = [i for i,j in enumerate(df.simid) if j == re.sub('_$' ,'',sim.name)]
			mat[df_index, :] = simphen[list(df['gene'].iloc[df_index]),:]
		self.stored_results[mutation_type + 'effect_uniqueGeneIds'] = pd.concat([df.reset_index(drop=True), pd.DataFrame(mat, columns = mat_colnames)], axis=1)
	def getNumberOfRegulatorsByGeneByCell(self):
		diftypes = sorted(set(self.netcontainer.networks[0][0].tipo[2][self.netcontainer.networks[0][0].tipo[3]==type1define['tf']]))
		tuplas = [[[re.sub('_$' ,'',net.simid)]*net.rownames.shape[0],[net.cell]*net.rownames.shape[0], net.rownames] + [np.concatenate([np.sum(net.mat[:, net.tipo[2][net.colnames]==t], axis=1).reshape([net.colnames.shape[0],1]) for t in diftypes], axis=1)] for i in self.netcontainer.networks for net in i]
		df= self.netcontainer.summaryTables['tfs_cells_sep'][['gene_unique_id', 'simset', 'simid', 'cell', 'gene', 'type0', 'type1','type2', 'type3', 'number']]
		## Check that data can be directly concatenated:
		t2 = np.concatenate([t[2] for t in tuplas], axis = 0)
		t3 = np.array([i for t in tuplas for i in t[0]])
		if(np.all(t2 == df.gene) & np.all(t3 ==  df.simid)):
			inv_map = {v: k for k, v in type1define.items()}
			df2 = pd.DataFrame(np.concatenate([tup[3] for tup in tuplas], axis = 0) , columns = [inv_map[t] for t in diftypes])
			df2 = pd.concat([df, df2], axis = 1)
		else:
			#Not equal! make loop step by step
			print("not equal order in getNumberOfRegulatorsByGeneByCell")
			#Not implemented
		df2['all_activators'] = df2.lineage_this_cell + df2.lineage_other_cell + df2.other_activator
		self.stored_results['regulatorsByGeneByCell']  = df2
		self.stored_results['regulatorsByGeneByCell_means']  = df2.groupby('type2').mean()
	def getMeanPhenotypeByGeneByCell(self, mut_type = 'tfs'):
		#df= self.netcontainer.summaryTables['tfs_cells_sep']['gene_unique_id', 'simset', 'simid', 'cell', 'gene', 'type0', 'type1','type2', 'type3', 'number']
		regtypes = sorted(set(self.netcontainer.networks[0][0].tipo[2][self.netcontainer.networks[0][0].tipo[3]==type1define['tf']]))
		tipo_by_cell = [self.netcontainer.networks[0][cell].tipo for cell in range(len(self.netcontainer.networks[0]))]
		inv_map = {v: k for k, v in type1define.items()}
		mean_phenotype = []
		for sim in self.simset.simulations:
			if(mut_type == 'tfs'):
				mut = sim.mutanttfs
				mutname = 'tf_mutated'
			else:
				mut = sim.mutantsites
				mutname = 'mutated_sites'
			wt=sim.expr.iloc[mut.index] #here .iloc is equivalent to .loc
			phenotype = mut[[i for i in mut.columns if 'exp' in i]] - wt[[i for i in wt.columns if 'exp' in i]]
			phenotype[mutname] = mut[mutname]
			#phenotype = phenotype.reset_index(drop=False)
			for cell in range(sim.ncells):
				for g in set(phenotype.index):
					aux_a =  [re.sub('s[0-9]+_*' ,'',sim.name), re.sub('_$' ,'',sim.name), cell, g] + [t[g] for t in tipo_by_cell[cell]]
					aux_b = []
					this_gene= phenotype.loc[g][['exp' + str(cell), mutname]].iloc[list(phenotype.loc[g]['exp' + str(cell)] != 0)]
					for r in regtypes:
						rr = this_gene.iloc[tipo_by_cell[cell][2][list(this_gene[mutname])] == r]
						if(rr.shape[0] > 0):
							aux_b.append(np.mean(np.array(rr[[i for i in rr.columns if 'exp' in i]])))
						else:
							aux_b.append(np.nan)
					all_activators = [True  if xx in [type1define['other_activator'], type1define['lineage_this_cell']] else False for xx in tipo_by_cell[cell][1][list(this_gene[mutname])]]
					rr = this_gene.iloc[all_activators]
					if(rr.shape[0] > 0):
						aux_b.append(np.mean(np.array(rr[[i for i in rr.columns if 'exp' in i]])))
					else:
						aux_b.append(np.nan)
					mean_phenotype.append(aux_a + aux_b)
		colnames = ['simset', 'simid', 'cell', 'gene'] + ['type' + str(i) for i in range(len(tipo_by_cell[0]))]	+ [inv_map[i] for i in regtypes] + ['all_activators']
		df= pd.DataFrame(mean_phenotype, columns = colnames)
		self.stored_results['meanPhenotypeByGene_' + mut_type]  = df
		self.stored_results['meanPhenotypeByGene_' + mut_type + '_means']  = df.groupby('type2').mean()

	def getCorrelationBetweenTerminalTypes(self):
		import itertools
		tipo_by_cell = [self.netcontainer.networks[0][cell].tipo for cell in range(len(self.netcontainer.networks[0]))]
		regtypes = sorted(set(self.netcontainer.networks[0][0].tipo[2][self.netcontainer.networks[0][0].tipo[3]==type1define['tf']]))
		tertypes = sorted(set(self.netcontainer.networks[0][0].tipo[2][self.netcontainer.networks[0][0].tipo[3]==type1define['non_tf']]))
		regnames = np.where(tipo_by_cell[0][3] == type1define['tf'])[0]
		ternames = np.where(tipo_by_cell[0][3] == type1define['non_tf'])[0]
		inv_map = {v: k for k, v in type1define.items()}
		typepairs = list(itertools.combinations_with_replacement(tertypes, 2))
		colnames = ['simset', 'simid', 'cell'] + ['Act_' + inv_map[a] + ':' + inv_map[b] if a != b else 'Act_' + inv_map[a] + '_selfcor' for a, b in typepairs] + ['Inh_' + inv_map[a] + ':' + inv_map[b] if a != b else 'Inh_' + inv_map[a] + '_selfcor' for a, b in typepairs]
		tuplas = []
		for sim in self.simset.simulations:
			for cell in range(sim.ncells):
				this_sim = [re.sub('s[0-9]+_*' ,'',sim.name),re.sub('_$' ,'',sim.name), cell]
				this_sim_inh = []
				mat=np.zeros([ternames.shape[0], regnames.shape[0]])
				mat_inh=np.zeros([ternames.shape[0], regnames.shape[0]])
				#Get regulators for each gene
				for ter_ind, ter in enumerate(ternames):
					mat[ter_ind, sim.getActivators(target=[ter], cell=cell)] = 1
					mat_inh[ter_ind, sim.getInhibitors(target=[ter], cell=cell)] = 1
				mat_cor = np.zeros([ternames.shape[0], ternames.shape[0]])
				mat_cor_inh = np.zeros([ternames.shape[0], ternames.shape[0]])
				#calculate correlations between each pair of genes
				for ter_ind in range(mat.shape[0]):
					for ter_ind2 in range(ter_ind):
						y = np.where(mat[ter_ind,:] + mat[ter_ind2,:] > 0)[0].shape[0]
						y_inh = np.where(mat_inh[ter_ind,:] + mat_inh[ter_ind2,:] > 0)[0].shape[0]
						if (y == 0):
							mat_cor[ter_ind, ter_ind2] = np.nan
						else:
							mat_cor[ter_ind, ter_ind2] = np.where(mat[ter_ind,:] + mat[ter_ind2,:] == 2)[0].shape[0]/y
						if (y_inh == 0):
							mat_cor_inh[ter_ind, ter_ind2] = np.nan
						else:
							mat_cor_inh[ter_ind, ter_ind2] = np.where(mat_inh[ter_ind,:] + mat_inh[ter_ind2,:] == 2)[0].shape[0]/y_inh
				mat_cor = mat_cor + np.transpose(mat_cor)
				mat_cor_inh = mat_cor_inh + np.transpose(mat_cor_inh)
				for a, b in typepairs:
					if(a != b):
						mean_cor = np.mean(mat_cor[np.where(tipo_by_cell[cell][2][ternames] == a)[0],:][:, np.where(tipo_by_cell[cell][2][ternames] == b)[0]])
						mean_cor_inh = np.mean(mat_cor_inh[np.where(tipo_by_cell[cell][2][ternames] == a)[0],:][:, np.where(tipo_by_cell[cell][2][ternames] == b)[0]])
					else:
						aux_mat = mat_cor[np.where(tipo_by_cell[cell][2][ternames] == a)[0],:][:, np.where(tipo_by_cell[cell][2][ternames] == b)[0]]
						aux_mat_inh = mat_cor_inh[np.where(tipo_by_cell[cell][2][ternames] == a)[0],:][:, np.where(tipo_by_cell[cell][2][ternames] == b)[0]]
						mask = np.ones(aux_mat.shape, dtype=bool)
						np.fill_diagonal(mask, 0)
						mean_cor = np.mean(aux_mat[mask])
						mean_cor_inh = np.mean(aux_mat_inh[mask])
					this_sim.append(mean_cor)
					this_sim_inh.append(mean_cor_inh)
				tuplas.append(this_sim + this_sim_inh)
		df = pd.DataFrame(tuplas, columns = colnames)
		self.stored_results['regCorrelationsBetweenTypes']  = df
	def getOverlapOfRegulationSets(self):
		import itertools
		tipo_by_cell = [self.netcontainer.networks[0][cell].tipo for cell in range(len(self.netcontainer.networks[0]))]
		regtypes = sorted(set(self.netcontainer.networks[0][0].tipo[2][self.netcontainer.networks[0][0].tipo[3]==type1define['tf']]))
		tertypes = sorted(set(self.netcontainer.networks[0][0].tipo[2][self.netcontainer.networks[0][0].tipo[3]==type1define['non_tf']]))
		regnames = np.where(tipo_by_cell[0][3] == type1define['tf'])[0]
		ternames = np.where(tipo_by_cell[0][3] == type1define['non_tf'])[0]
		inv_map = {v: k for k, v in type1define.items()}
		typepairs = list(itertools.permutations(tertypes, 2))
		colnames = ['simset', 'simid', 'cell'] + ['Act_' + inv_map[a] + ':' + inv_map[b] for a, b in typepairs] + ['Inh_' + inv_map[a] + ':' + inv_map[b] for a, b in typepairs]
		tuplas = []
		for sim in self.simset.simulations:
			for cell in range(sim.ncells):
				this_sim = [re.sub('s[0-9]+_*' ,'',sim.name),re.sub('_$' ,'',sim.name), cell]
				this_sim_inh = []
				for ta, tb in typepairs:
					genes_a = [g for g in ternames if tipo_by_cell[cell][2][g] == ta]
					genes_b = [g for g in ternames if tipo_by_cell[cell][2][g] == tb]
					aa=[]
					ai=[]
					ba=[]
					bi=[]
					for g in genes_a:
						aa = aa +sim.getActivators(target=[g], cell=cell)
						ai = ai + sim.getInhibitors(target=[g], cell=cell)
					aa= set(aa)
					ai = set(ai)
					for g in genes_b:
						ba = ba +sim.getActivators(target=[g], cell=cell)
						bi = bi + sim.getInhibitors(target=[g], cell=cell)
					ba= set(ba)
					bi = set(bi)
					if(len(aa)>0):
						overlap_act = len(aa.intersection(ba))/len(aa)
					else:
						overlap_act = np.nan
					if(len(ai)>0):
						overlap_inh = len(ai.intersection(bi))/len(ai)
					else:
						overlap_inh = np.nan
					this_sim.append(overlap_act)
					this_sim_inh.append(overlap_inh)
				tuplas.append(this_sim + this_sim_inh)
		df = pd.DataFrame(tuplas, columns = colnames)
		self.stored_results['regIntersectionBetweenTypes']  = df
	def writeTables(self, extraname = ''):
		#simset = re.sub('s[0-9]+_*' ,'',self.simset.simulations[0].name) #should be the same as current line
		if(extraname != ''):
			extraname = '_' + extraname
		simset =  self.netcontainer.set_name  + extraname
		for name, table in self.stored_results.items():
			if('skeleton' in name):
				continue
			table.to_csv(simset + '_' + name + '.csv')
		#self.saveSelf(extraname)
	def saveSelf(self, name=''):
		import _pickle as pck
		filename = self.netcontainer.set_name + name +'_extraDataObject.pck'
		with open(filename, 'wb') as output:
 			pickler = pck.Pickler(output, -1)
 			pickler.dump(self)
	@staticmethod
	def load(filenamein):
		with open(filenamein, 'rb') as f:
			print('opening ' + filenamein)
			import _pickle as pck
			dex = pck.load(f)
		return dex
def main():
	#default_path = '../cellevolver_shared/'
	#all_conditions = ['4cell_mce0', '4cell_mce0fix_mutb', '4cell_mce1fix', '4cell_mce2fix', '4cell_mce2inh5', '4cell_mce0fix', '4cell_mce0_mutb', '4cell_mce1fix_mutb', '4cell_mce2fix_mutb', '4cell_mce2inh5_mutb', '4cell_mce0Xss']
	default_path= './sim_tables/'
	all_conditions=['4cell_mce4fix', '4cell_mce5fix', '4cell_mce8fix', '4cell_mce9fix', '6cell_mce6fix', '6cell_mce7fix']

	path = default_path
	conditions = all_conditions
	mm = multiSetMotifFinder(path, conditions,  type_to_separate = 1, type_to_separate_motifs=3, selectBy=0, starting_n = 3, functionalFilter = True, hashSearch=True, randsize=10, ignoreSelfReg = True)
	mm.fullMotifAnalysis(extraname = sys.argv[1])

if __name__ == "__main__":
    main()


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import torch
import ctypes
import logging

from amd.rali.pipeline import Pipeline
import amd.rali.ops as ops
import amd.rali.types as types

import sys
import numpy as np
from enum import Enum
import cv2
import time

class COCOPipeline(Pipeline):
    def __init__(self, batch_size, num_threads, device_id, data_dir, ann_dir, crop, rali_cpu=True):
        super(COCOPipeline, self).__init__(batch_size, num_threads,
                                           device_id, seed=12 + device_id, rali_cpu=rali_cpu)
        self.input = ops.COCOReader(
            file_root=data_dir, annotations_file=ann_dir)
        rali_device = 'cpu' if rali_cpu else 'gpu'
        decoder_device = 'cpu' if rali_cpu else 'mixed'
        # device_memory_padding = 211025920 if decoder_device == 'mixed' else 0
        # host_memory_padding = 140544512 if decoder_device == 'mixed' else 0
        # self.decode = ops.ImageDecoderRandomCrop(device=decoder_device, output_type=types.RGB,
        # 											device_memory_padding=device_memory_padding,
        # 											host_memory_padding=host_memory_padding,
        # 											random_aspect_ratio=[0.8, 1.25],
        # 											random_area=[0.1, 1.0],
        # 											num_attempts=100)
        self.decode = ops.ImageDecoder(
            device=decoder_device, output_type=types.RGB)
        self.crop = ops.SSDRandomCrop(num_attempts=5)
        self.res = ops.Resize(device=rali_device, resize_x=crop, resize_y=crop)
        self.twist = ops.ColorTwist(device=rali_device)
        self.cmnp = ops.CropMirrorNormalize(device="gpu",
                                            output_dtype=types.FLOAT,
                                            output_layout=types.NCHW,
                                            crop=(crop, crop),
                                            image_type=types.RGB,
                                            mirror=0,
                                            mean=[0.485 * 255, 0.456 *
                                                  255, 0.406 * 255],
                                            std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
        # Random variables
        self.rng1 = ops.Uniform(range=[0.5, 1.5])
        self.rng2 = ops.Uniform(range=[0.875, 1.125])
        self.rng3 = ops.Uniform(range=[-0.5, 0.5])
        print('rali "{0}" variant'.format(rali_device))

    def define_graph(self):
        saturation = self.rng1()
        contrast = self.rng1()
        brightness = self.rng2()
        hue = self.rng3()
        self.jpegs, self.bb, self.labels = self.input(name="Reader")
        images = self.decode(self.jpegs)
        images = self.crop(images)
        images = self.res(images)
        images = self.twist(images, saturation=saturation,
                            contrast=contrast, brightness=brightness, hue=hue)
        output = self.cmnp(images)
        return [output, self.bb, self.labels]

class RALICOCOIterator(object):
    """
    COCO RALI iterator for pyTorch.

    Parameters
    ----------
    pipelines : list of amd.rali.pipeline.Pipeline
                List of pipelines to use
    size : int
           Epoch size.
    """

    def __init__(self, pipelines, tensor_layout=types.NCHW, reverse_channels=False, multiplier=[1.0, 1.0, 1.0], offset=[0.0, 0.0, 0.0], tensor_dtype=types.FLOAT):

        # self._num_gpus = len(pipelines)
        assert pipelines is not None, "Number of provided pipelines has to be at least 1"

        self.loader = pipelines
        self.tensor_format = tensor_layout
        self.multiplier = multiplier
        self.offset = offset
        self.reverse_channels = reverse_channels
        self.tensor_dtype = tensor_dtype
        self.bs = self.loader._batch_size
        self.w = self.loader.getOutputWidth()
        self.h = self.loader.getOutputHeight()
        self.n = self.loader.getOutputImageCount()
        self.rim = self.loader.getRemainingImages()
        print("____________REMAINING IMAGES____________:", self.rim)
        color_format = self.loader.getOutputColorFormat()
        self.p = (1 if color_format is types.GRAY else 3)
        if self.tensor_dtype == types.FLOAT:
            self.out = np.zeros(
                (self.bs*self.n, self.p, int(self.h/self.bs), self.w,), dtype="float32")
        elif self.tensor_dtype == types.FLOAT16:
            self.out = np.zeros(
                (self.bs*self.n, self.p, int(self.h/self.bs), self.w,), dtype="float16")

    def next(self):
        return self.__next__()

    def __next__(self):
        print("In the next routine of COCO Iterator")
        if(self.loader.isEmpty()):
            timing_info = self.loader.Timing_Info()
            print("Load     time ::", timing_info.load_time)
            print("Decode   time ::", timing_info.decode_time)
            print("Process  time ::", timing_info.process_time)
            print("Transfer time ::", timing_info.transfer_time)
            raise StopIteration

        if self.loader.run() != 0:
            raise StopIteration
        self.lis = []  # Empty list for bboxes
        self.lis_lab = []  # Empty list of labels

        if(types.NCHW == self.tensor_format):
            self.loader.copyToTensorNCHW(
                self.out, self.multiplier, self.offset, self.reverse_channels, int(self.tensor_dtype))
        else:
            self.loader.copyToTensorNHWC(
                self.out, self.multiplier, self.offset, self.reverse_channels, int(self.tensor_dtype))

        self.img_names_length = np.empty(self.bs, dtype="int32")
        self.img_names_size = self.loader.GetImageNameLen(self.img_names_length)
        print("Image name length:",self.img_names_size)
# Images names of a batch
        self.Img_name = self.loader.GetImageName(self.img_names_size)
        print("Image names in a batch ",self.Img_name)
#Count of labels/ bboxes in a batch
        self.bboxes_label_count = np.zeros(self.bs, dtype="int32")
        self.count_batch = self.loader.GetBoundingBoxCount(self.bboxes_label_count)
        print("Count Batch:",self.count_batch)
# 1D labels array in a batch
        self.labels = np.zeros(self.count_batch, dtype="int32")
        self.loader.GetBBLabels(self.labels)
        print(self.labels)
# 1D bboxes array in a batch
        self.bboxes = np.zeros((self.count_batch*4), dtype="float32")
        self.loader.GetBBCords(self.bboxes)
        print(self.bboxes)
#Image sizes of a batch
        self.img_size = np.zeros((self.bs * 2),dtype = "int32")
        self.loader.GetImgSizes(self.img_size)
        print("Image sizes:",self.img_size)
        count =0
        sum_count=0
        for i in range(self.bs):
            count = self.bboxes_label_count[i]
            print("labels:",self.labels[sum_count : sum_count+count])
            print("bboxes:",self.bboxes[sum_count*4 : (sum_count+count)*4])
            print("Image w & h:",self.img_size[i*2:(i*2)+2])
            print("Image names:",self.Img_name[i*16:(i*16)+12])
            self.img_name = self.Img_name[i*16:(i*16)+12]
            self.img_name=self.img_name.decode('utf_8')
            self.img_name = np.char.lstrip(self.img_name, chars ='0')
            print("Image names:",self.img_name)
            self.label_2d_numpy = (self.labels[sum_count : sum_count+count])
            self.label_2d_numpy = np.reshape(self.label_2d_numpy, (-1, 1)).tolist()
            self.bb_2d_numpy = (self.bboxes[sum_count*4 : (sum_count+count)*4])
            self.bb_2d_numpy = np.reshape(self.bb_2d_numpy, (-1, 4)).tolist()
            self.lis_lab.append(self.label_2d_numpy)
            self.lis.append(self.bb_2d_numpy)
            sum_count = sum_count +count

        self.target = self.lis
        self.target1 = self.lis_lab

        max_cols = max([len(row) for batch in self.target for row in batch])
        max_rows = max([len(batch) for batch in self.target])
        self.bb_padded = [
            batch + [[0] * (max_cols)] * (max_rows - len(batch)) for batch in self.target]
        self.bb_padded = torch.FloatTensor(
            [row + [0] * (max_cols - len(row)) for batch in self.bb_padded for row in batch])
        self.bb_padded = self.bb_padded.view(-1, max_rows, max_cols)
        # print(self.bb_padded)

        max_cols1 = max([len(row) for batch in self.target1 for row in batch])
        max_rows1 = max([len(batch) for batch in self.target1])
        self.labels_padded = [
            batch + [[0] * (max_cols1)] * (max_rows1 - len(batch)) for batch in self.target1]
        self.labels_padded = torch.LongTensor(
            [row + [0] * (max_cols1 - len(row)) for batch in self.labels_padded for row in batch])
        self.labels_padded = self.labels_padded.view(-1, max_rows1, max_cols1)
        # print(self.labels_padded)

        if self.tensor_dtype == types.FLOAT:
            return torch.from_numpy(self.out), self.bb_padded, self.labels_padded
        elif self.tensor_dtype == types.FLOAT16:
            return torch.from_numpy(self.out.astype(np.float16)), self.bb_padded, self.labels_padded

    def reset(self):
        self.loader.raliResetLoaders()

    def __iter__(self):
        self.loader.raliResetLoaders()
        return self

def main():
    if len(sys.argv) < 5:
        print('Please pass the folder image_folder Annotation_file cpu/gpu batch_size')
        exit(0)

    image_path = sys.argv[1]
    ann_path = sys.argv[2]
    if(sys.argv[3] == "cpu"):
        _rali_cpu = True
    else:
        _rali_cpu = False
    bs = int(sys.argv[4])
    nt = 1
    di = 0
    crop_size = 224
    pipe = COCOPipeline(batch_size=bs, num_threads=nt, device_id=di,
                        data_dir=image_path, ann_dir=ann_path, crop=crop_size, rali_cpu=_rali_cpu)
    pipe.build()
    imageIterator = RALICOCOIterator(
        pipe, multiplier=pipe._multiplier, offset=pipe._offset)
    for i, it in enumerate(imageIterator, 0):
        print("**************", i, "*******************")
        print("**************starts*******************")
        print("\nBBOXES:\n", it[1])
        print("\nLABELS:\n", it[2])
        print("**************ends*******************")
        print("**************", i, "*******************")

if __name__ == '__main__':
    main()


from __future__ import division
from __future__ import print_function
from __future__ import absolute_import

import logging
import sys
import os
from datetime import datetime
from random import shuffle
import numpy as np

from nn import nn_layer
from nn import conv_layer
from nn import pooling_layer
from nn import activation
from nn import simple_nn
from util import mnist

def get_kernels():
    result = []
    uuid = 1

    # 1. 3x3 kernels
    for i in range(8):
        func = activation.reluFunc
        if i % 3 == 0:
            func = activation.tanhFunc
        kernel = conv_layer.Kernel(3, func, uuid)
        result.append(kernel)
        uuid += 1

    # 2. 5x5 kernels
    for i in range(8):
        func = activation.reluFunc
        if i % 3 == 0:
            func = activation.tanhFunc
        kernel = conv_layer.Kernel(5, func, uuid)
        result.append(kernel)
        uuid += 1
    return result

def construct_cnn(l2=0.0):
    img_input = nn_layer.InputLayer("mnist_input", 784)
    output_layer = nn_layer.SoftmaxOutputLayer("mnist_output", 10)

    # 1. set input and output layers
    nn = simple_nn.NNetwork()
    nn.set_input(img_input)
    nn.set_output(output_layer)

    # 2. add Conv-Pooling layers
    c1 = conv_layer.ConvLayer("conv1")
    c1.set_kernels(get_kernels())
    nn.add_hidden_layer(c1)

    # 2x2 none-overlapping max-pooling
    p1 = pooling_layer.MaxPoolingLayer("pool1", 2, 2)
    nn.add_hidden_layer(p1)

    # 3. add some full-connected hidden layers
    h1 = nn_layer.HiddenLayer("h1", 512, activation.tanhFunc)
    h1.set_lambda2(l2)
    nn.add_hidden_layer(h1)

    h2 = nn_layer.HiddenLayer("h2", 128, activation.tanhFunc)
    h2.set_lambda2(l2)
    nn.add_hidden_layer(h2)

    h3 = nn_layer.HiddenLayer("h3", 10, activation.reluFunc)
    h3.set_lambda2(l2)
    nn.add_hidden_layer(h3)

    # 3. complete nn construction
    # print("%s" % (nn))
    fake_img = np.zeros((1, 28, 28))
    img_input.feed(fake_img)
    nn.connect_layers()
    logging.info("NN information:\n" + nn.get_detail())
    return nn

def train_it(nn, train_data, lr):
    labels = train_data[0]
    imgs = train_data[1]

    # shuffle the data
    alist = list(range(labels.shape[0]))
    shuffle(alist)

    num = 0
    for i in alist:
        label = labels[i, :]
        img = imgs[i, :]
        nn.train(img, label, lr)
        if num % 1000 == 0:
            logging.info("num=%d" % num)
        num += 1

    return

def evaluate_it(nn, test_data, prefix):
    labels = test_data[0]
    imgs = test_data[1]
    num = labels.shape[0]

    total_correct = 0
    total_cost = 0

    for i in range(num):
        label = labels[i, :]
        img = imgs[i, :]
        correct, cost = nn.evaluate(img, label)
        total_correct += correct
        total_cost += cost

    accuracy = float(total_correct) / num
    avg_cost = total_cost/num

    msg = "[%s] accuracy=%.4f, avg_cost=%.4f" % (prefix, accuracy, avg_cost)
    logging.info(msg)
    return

def get_lr(step, current_lr):
    """simple learning rate scheduler"""
    lrs = {0: 0.008, 1: 0.005, 2: 0.003, 5: 0.002, 6: 0.001, 8: 0.0008, 10: 0.0005, 15: 0.0001}
    if step in lrs:
        return lrs[step]
    return current_lr

def train_nn(data_dir):
    nn = construct_cnn()
    # l2 = 1e-3
    # nn = construct_big_nn(l2)
    train_data = mnist.load_data_3d(data_dir, "train")
    test_data = mnist.load_data_3d(data_dir, "test")
    if (train_data is None) or (test_data is None):
        logging.error("[ERROR] failed to load data")
        return

    lr = 0.005
    for i in range(100):
        lr = get_lr(i, lr)
        msg = "begin epoch-%s, lr=%.6f" % (i, lr)
        logging.info(msg)
        train_it(nn, train_data, lr)
        evaluate_it(nn, test_data, "test")
        evaluate_it(nn, train_data, "train")
        logging.info("end epoch-%s" % (i, ))

    return

def main():
    train_nn("./data/")
    return

def setup_log():
    logfile = "./train.%s.log" % (os.getpid())
    if len(sys.argv) > 1:
        logfile = sys.argv[1]
    print("logfile=%s" % (logfile,))
    logging.basicConfig(filename=logfile, format='[%(asctime)s] %(message)s', level=logging.DEBUG)
    return

if __name__ == "__main__":
    setup_log()
    main()


from django.shortcuts import render
from .models import Alumni
# Create your views here.
def alumni(request):
    obj_alumni = Alumni.objects.all()
    obj_year = []
    for i in obj_alumni:obj_year.append(i.year)
    new_lst = list(set(obj_year))
    new_lst.sort()
    dict = {'alum':obj_alumni,'year':new_lst}
    return render(request,'alumni.html',context=dict)


import sys
import csv
import json
from http.server import HTTPServer, SimpleHTTPRequestHandler, test as test_orig

def leArquivo( arquivo ):
	with open( arquivo, newline='' ) as csvfile:
		listaProdutos = csv.reader( csvfile, delimiter=',' )
		novaLista=[]
		for row in listaProdutos:
			novaLista.append( row )

	return novaLista

def arquivoJSON( arquivo ):
	return json.dumps( leArquivo( arquivo ))

def test (*args):
   test_orig(*args, port=8000)

class CORSRequestHandler( SimpleHTTPRequestHandler ):
    def end_headers (self):
        self.send_header( 'Access-Control-Allow-Origin', '*' )
        self.send_header( 'Content-Type', 'application/json' )
        SimpleHTTPRequestHandler.end_headers(self)

if __name__ == '__main__':
	arq = open( 'index.html','w+' )
	arq.write( arquivoJSON( 'bancodedados.csv' ))
	arq.close()
	test( CORSRequestHandler, HTTPServer )


from display import *
from draw import *
from matrix import *

screen = new_screen()
color = [ 255, 100, 0 ]
matrix = []
y = 10

for x in range(y):
	x = x + 1
	add_edge(matrix, 0, 0, 0, int(XRES * x / y), YRES , 0)
for x in range(y):
	x = x + 1
	add_edge(matrix, 0, 0, 0, XRES, int(YRES * x / y), 0)

draw_lines( matrix, screen, color )

color = [ 0, 100, 255 ]
matrix = []

for x in range(y):
	x = x + 1
	add_edge(matrix, XRES, YRES, 0, int(XRES * x / y), 0 , 0)
for x in range(y):
	x = x + 1
	add_edge(matrix, XRES, YRES, 0, 0, int(XRES * x / y), 0)

draw_lines( matrix, screen, color )

display(screen)


from collections import deque
class TreeNode(object):
    def __init__(self, x):
        self.val = x
        self.left = None
        self.right = None
class Solution(object):
    def postorderTraversal(self, root):
        """
        :type root: TreeNode
        :rtype: List[int]
        """
        res=[]
        queue=deque()
        lastVisit=root
        while root or queue:
            while root:
                queue.append(root)
                root=root.left
            #先一直往左找，因为先返回的是左
            root =queue[-1]
            if root.right is None or root.right == lastVisit:
                res.append(root.val)
                queue.pop()
                lastVisit=root
                root=None
                #对当前节点判断一下，是该往上走了还是该往右走了，如果
            else:
                root=root.right
        return res


from django.shortcuts import render
from django.contrib.auth.decorators import login_required
from .models import House
from django.http import Http404
from django.views import generic
from carts.models import Cart
from math import ceil
from landlords.models import Landlord
from .choices import number_of_bedrooms,price, district_choices,division_choices,zones_choices
from rest_framework import serializers

class HomeView(generic.ListView):

    template_name = "houses/home.html"
    def get_queryset(self, *args, **kwargs):
        request= self.request

        # print (request.session.get("first_name", "Unknown"))
        return  House.objects.featured()[:3]

    def get_context_data(self, *args, **kwargs):
            context = super(HomeView, self).get_context_data(*args, **kwargs)
            request = self.request
            cart_obj, new_obj = Cart.objects.new_or_get(request)
            context['cart'] = cart_obj
            return context

def home_view(request):
    houses = House.objects.featured()[:3]

    context = {"houses":houses}
    template = "houses/home.html"
    return render(request, template, context)

class  DetailSlugView(generic.DetailView):
        model = House
        queryset = House.objects.all()
        template_name = "houses/detail.html"

        def get_context_data(self, *args, **kwargs):
            context = super(DetailSlugView, self).get_context_data(*args, **kwargs)
            request = self.request
            cart_obj, new_obj = Cart.objects.new_or_get(request)
            context['cart'] = cart_obj
            return context

        def get_object(self, *args, **kwargs):
            request = self.request
            slug    = self.kwargs.get('slug')
            try:
                instance = House.objects.get(slug=slug, active=True)
            except House.DoesNotExist:
                raise Http404("This House either doesnot exsit or it was deleted")
            except House.MultipleObjectsReturned:
                qs = House.Objects.filter(slug=slug, active=True)
                instance = qs.first()
            except:
                raise Http404
            return instance

def about(request):
    landlord = Landlord.objects.order_by('-name')

    # Get MVP
    mvp_landlords = Landlord.objects.all().filter(is_mvp=True)
    context = {'landlord':landlord,
                'mvp_landlords':mvp_landlords
    }
    template = 'houses/about.html'
    return render(request, template, context)

@login_required
def userProfile(request):
    # qs = House.objects.all()
    instance_obj = House.objects.all()
    district_exact_query = request.GET.get('district_exact')
    division_exact_query = request.GET.get('division_exact')
    zone_exact_query     = request.GET.get('zone_exact')
    price_exact_query    =  request.GET.get(' price_exact')
    rooms_exact_query    =   request.GET.get('rooms_exact')

    if district_exact_query != '' and  district_exact_query is not None:
        instance_obj = instance_obj.filter(District__icontains=district_exact_query)

    if division_exact_query != '' and  division_exact_query is not None:
        instance_obj = instance_obj.filter(Division__icontains=division_exact_query)

    if zone_exact_query != '' and  zone_exact_query is not None:
        instance_obj = instance_obj.filter(Zone__icontains= zone_exact_query)

    if price_exact_query != '' and  price_exact_query is not None:
        instance_obj = instance_obj.filter(Price__icontains= price_exact_query)

    if rooms_exact_query != '' and  rooms_exact_query is not None:
        instance_obj = instance_obj.filter(Number_Of_Rooms__icontains= rooms_exact_query)

    user = request.user
    context ={
        'user':user,
        'instance_obj':instance_obj,

        }
    template = 'houses/profile.html'
    return render(request, template, context)

def search(request):
    queryset_list = House.objects.all()

    #   keywords
    if 'district' in request.GET:
        district = request.GET['district']
        if district:
            queryset_list = queryset_list.filter(District__iexact=district)

    # Division
    if 'division' in request.GET:
        division = request.GET['division']
        if division:
            queryset_list = queryset_list.filter(Division__iexact=division)

    # Zone
    if 'zone' in request.GET:
        zone = request.GET['zone']
        if zone:
            queryset_list= queryset_list.filter(Zone__iexact=zone)

    # Price
    if 'price' in request.GET:
        price = request.GET['price']
        if price:
            queryset_list = queryset_list.filter(Price__Ite=price)

    # Bedrooms
    if 'bedrooms' in request.GET:
        bedrooms = request.GET['bedrooms']
        if bedrooms:
            queryset_list = queryset_list.filter(Number_Of_Rooms__Ite=bedrooms)

    context = {
        'number_of_bedrooms':number_of_bedrooms,
        'price':price,
        'district_choices':district_choices,
        'division_choices':division_choices,
        'zones_choices ':zones_choices ,

    }
    return render(request, 'search/views.html', context)


#Ingresar n° pos, <500, informar si en n° primo

n = int(input('Valor: '))
while n >= 500 or n <= 0:
    n = int(input('Valor: '))

def primo(n):
    primo = False
    for i in range(1, n, 1):
        if n%i == 0:
            if n != i and i != 1:
                primo = True
    if primo:
        return 'No es primo'
    else:
        return 'Primo'

print(primo(n))


from OpenOrange import *

class ReportWindow(Window):

    def replaceAccountsShortCut(self):
        from Account import Account
        record = self.getRecord()
        rngs = filter(lambda x: bool(x), record.Account.split(","))
        x_rngs = []
        for rng in rngs:
            accs = filter(lambda x: bool(x), rng.split(":"))
            x_accs = []
            for acc in accs:
                account = Account.bring(acc)
                if account and account.Code != acc: acc = account.Code
                x_accs.append(acc)
            x_rngs.append(":".join(x_accs))
        record.Account = ",".join(x_rngs)

    def afterEdit(self, fieldname):
        Window.afterEdit(self, fieldname)
        if fieldname == "Account":
            record = self.getRecord()
            if record.Account and not (record.Account.strip().endswith(":") or record.Account.strip().endswith(",")):
                self.replaceAccountsShortCut()


import sys
from math import *

pdb_filename = sys.argv[1]
chain1 = sys.argv[2] #A
chain2 = sys.argv[3] #B
threshold = float(sys.argv[4])
all_A = False
if len(sys.argv)>5:
    if sys.argv[5] == 'all':
        all_A = True

def euclidean_d(atom1,atom2):
    d = sqrt(((atom1[0]-atom2[0])**2)+((atom1[1] - atom2[1])**2)+((atom1[2]-atom2[2])**2))
    return d

atoms1 = []
atoms2 = []
with open(pdb_filename) as pdb:
    for line in pdb:
        if line[0:4] == 'ATOM':
            if all_A or line[12:16].strip() == 'CA':
                x = float(line[30:38].strip())
                y = float(line[38:46].strip())
                z = float(line[46:54].strip())
                res_num = line[22:26].strip()
                res_name = line[17:20].strip()
                if line[21] == chain1:
                    atoms1.append((res_num,res_name,(x, y, z)))
                elif line[21] == chain2:
                    atoms2.append((res_num,res_name,(x, y, z)))

for atom1 in atoms1:
    for atom2 in atoms2:
        if euclidean_d(atom1[2],atom2[2]) <= threshold:
            print('%1s %4s %3s %8.3f %8.3f %8.3f  |  %1s %4s %3s %8.3f %8.3f %8.3f'
                  % (chain1, atom1[0], atom1[1], atom1[2][0], atom1[2][1], atom1[2][2],
                     chain2, atom2[0], atom2[1], atom2[2][0], atom2[2][1], atom2[2][2]))


import my_functions
import pandas as pd
import seaborn as sns
from sklearn.metrics import confusion_matrix, classification_report, accuracy_score, balanced_accuracy_score, roc_curve, auc, f1_score, precision_score, recall_score, cohen_kappa_score
import matplotlib.pyplot as plt

def calculate_f1score(input_file):
    """
    Compute F1 weighted for training and validation

    Arguments:
    input_file (str) -- Path to .csv that has the predictions with columns and Prediction, True_Label, Split

    Returns:
    f1_train (float) -- f1 weighted of training
    f1_val (float) -- f1 weighted of validation
    """
    # Read data and splits into training and vallidation
    df = pd.read_csv(input_file)
    df_train = df[ df["split"] == "training" ]
    df_val = df[ df["split"] == "validation" ]

    # Compute F1 Weighted Training
    y_pred = df_train["Prediction"]
    y_true = df_train["True_Label"]
    f1_train = f1_score(y_true, y_pred, average = "weighted")

    # Compute F1 Weighted Validation
    y_pred = df_val["Prediction"]
    y_true = df_val["True_Label"]
    f1_val = f1_score(y_true, y_pred, average = "weighted")

    return f1_train, f1_val

def calculate_satisfying_metric(input_file):
    """
    Compute auc, balanced accuracy, precision, recall for training and validation

    Arguments:
    input_file (str) -- Path to .csv that has the predictions with columns and Prediction, True_Label, Split
    """
    # Read data and splits into training and vallidation
    df = pd.read_csv(input_file)
    df_train = df[ df["split"] == "training" ]
    df_val = df[ df["split"] == "validation" ]

    # Training
    # Divide y_pred and y_true
    y_pred = df_train["Prediction"]
    y_true = df_train["True_Label"]

    # Compute Satisfying Metrics
    baccuracy_train = balanced_accuracy_score(y_true, y_pred)
    cohen_train = cohen_kappa_score(y_true, y_pred, weights="linear")

    # Validation
    # Compute F1 Weighted
    y_pred = df_val["Prediction"]
    y_true = df_val["True_Label"]

    # Compute Satisfying Metrics
    baccuracy_val = balanced_accuracy_score(y_true, y_pred)
    cohen_val = cohen_kappa_score(y_true, y_pred, weights="linear")

    return baccuracy_train, cohen_train, baccuracy_val, cohen_val

def make_predictions(input_file, model, format_type, index_column = "PassengerId", target_column = "Survived"):
    """
    Make predictions using a sklearn model and put the true label in the same df

    Arguments:
    input_file (str) -- Path to .csv that has the features of the previously trained model
    model -- Sklearn model previously trained
    format_type (str) -- Format type of the save data (csv or pickle)
    index_column -- Index column of the dataset
    target_column -- Target column of the dataset

    Returns:
    df -- Pandas Dataframe with three columns index, predictions and the true label
    """
    # Read the data
    if format_type == "csv":
        df = pd.read_csv(input_file).set_index(index_column)
    elif format_type == "pickle":
        df = pd.read_pickle(input_file)

    df.rename(columns = {"Survived":"True_Label"}, inplace = True) # To have a more meaningful name

    # Create X (train values with features already computed)
    X, _ = my_functions.create_features_target(input_file, target_column = target_column, index_column = index_column, format_type = format_type)

    # Make predictions
    y_true = df["True_Label"]
    y_pred = pd.DataFrame(model.predict(X), index=df.index, columns = ["Prediction"])

    df = pd.concat([y_pred, y_true], axis=1)

    # Label data into live and die to have a more meaningful name
    df["True_Label"] = df["True_Label"].apply(lambda x: 'die' if x == 0 else 'live')
    df["Prediction"] = df["Prediction"].apply(lambda x: 'die' if x <= 0.5 else 'live') #  It is set like this so it works for models that return probabilities, 0.5 serves as a threshold

    return df

def save_predictions(train_file, val_file, output_file, model, format_type, index_column = "PassengerId", target_column = "Survived"):
    """
    Make predictions on the train and validation data and saves it to a .csv

    Arguments:
    train_file (str) -- Path to .csv that has the features
    val_file (str) -- Path to .csv that has the features
    output_file (str) -- Path to .csv where the predictions are going to be saved
    index_column -- Index column of the dataset
    target_column -- Target column of the dataset
    """

    # Make predictions for training
    df_train = make_predictions(train_file, model, format_type, index_column, target_column)
    df_train["split"] = "training"

    # Make predictions for validation
    df_val = make_predictions(val_file, model, format_type, index_column, target_column)
    df_val["split"] = "validation"

    df = df_train.append(df_val)
    df.to_csv(output_file)

# Metrics Plots

def plot_roc_curve(model, X, y, title = "Receiver Operating Characteristic"):
    """
    Plot ROC curve

    Arguments:
    model -- Sklearn model already train
    X -- Data to predict on. The features has to be the same with which the model was trained on
    y -- target value of the X data
    title (str) -- title of the new plot

    Returns:
    matplotlib ROC curve
    """

    probs = model.predict_proba(X)
    preds = probs[:,1]
    fpr, tpr, threshold = roc_curve(y, preds) # calculate the fpr and tpr for all thresholds of the classification
    roc_auc = auc(fpr, tpr)

    # Graph
    plt.title(title)
    plt.plot(fpr, tpr, 'b', label = 'AUC = %0.2f' % roc_auc)
    plt.legend(loc = 'lower right')
    plt.plot([0, 1], [0, 1],'r--')
    plt.xlim([0, 1])
    plt.ylim([0, 1])
    plt.ylabel('True Positive Rate')
    plt.xlabel('False Positive Rate')
    plt.show()

def plot_confusion_matrix(input_file, split):
    """
    Plot Confusion Matrix

    Arguments:
    input_file (str) -- Path to .csv that has the predictions with columns and Prediction, True_Label, Split
    split (str) -- training or validation
    Returns:
    Confusion Matrix matplotlib plot
    """

    df = pd.read_csv(input_file)
    df = df[df['split'] == split]

    labels = ['live', 'die']

    cm = confusion_matrix(df.True_Label, df.Prediction, normalize = 'true')
    cm = cm*100 # Multiply by 100 so te percentage is 99.7% instead of 0.997%
    cm = pd.DataFrame(cm, index=labels, columns=labels)

    # Customize heatmap (Confusion Matrix)
    sns.set(font_scale = 1.5)
    ax = sns.heatmap(cm, cmap='BuGn', annot=True, annot_kws={"size": 18,"weight":"bold"}, cbar=False, fmt='.3g')
    for t in ax.texts: t.set_text(t.get_text() + " %") # Put percentage in confusion matrix
    plt.xlabel('Predicted',weight='bold')
    plt.ylabel('Known',weight='bold')
    plt.yticks(rotation=0)
    plt.xticks(rotation=0)

def plot_confusion_matrices(input_file):
    """
    Plot Confusion Matrices of training and validation set

    Arguments:
    input_file (str) -- Path to .csv that has the predictions with columns and Prediction, True_Label, Split

    Returns:
    Confusion Matrix matplotlib plot
    """

    plt.rcParams['figure.figsize'] = [18, 5]
    plt.subplot(1,2,1)
    plot_confusion_matrix(input_file, "training")
    plt.title("Matrix Training")
    plt.subplot(1,2,2)
    plot_confusion_matrix(input_file, "validation")
    plt.title("Matrix Validation")

def plot_classification_report(input_file, split):
    """
    Plot Classification Report of sklearn

    Arguments:
    input_file (str) -- Path to .csv that has the predictions with columns and Prediction, True_Label, Split
    split (str) -- training or validation
    Returns:
    Classification Report plot
    """

    df = pd.read_csv(input_file)
    df = df[df['split'] == split]

    # Compute sklearn Classification Report and saves it in a pandas dataframe
    report = pd.DataFrame(classification_report(df.True_Label, df.Prediction, digits=3, output_dict=True)).transpose()
    report = report.loc[:, ["precision", "recall", "f1-score"]].drop(['accuracy', "macro avg"]) # Select what parts of the classification report to plot
    report = report*100 # Multiply by 100 so te percentage is 99.7% instead of 0.997%

    # Customize heatmap (Classification Report)
    sns.set(font_scale = 1.3)
    rdgn = sns.diverging_palette(h_neg=10, h_pos=130, s=80, l=62, sep=3, as_cmap=True)
    ax=sns.heatmap(report, cmap=rdgn, annot=True, annot_kws={"size": 14}, cbar=True, fmt='.3g', cbar_kws={'label':'%'}, center=90, vmin=0, vmax=100)
    ax.xaxis.tick_top()
    for t in ax.texts: t.set_text(t.get_text() + " %") #Put percentage
    plt.yticks(rotation = 0)

def plot_classification_reports(input_file):
    """
    Plot Classification Reports of training and validation set

    Arguments:
    input_file (str) -- Path to .csv that has the predictions with columns and Prediction, True_Label, Split
    Returns:
    Classification Report plot
    """

    plt.rcParams['figure.figsize'] = [18, 5]
    plt.subplot(1,2,1)
    plot_classification_report(input_file, split="training")
    plt.title("Training")
    plt.subplot(1,2,2)
    plot_classification_report(input_file, split="validation")
    plt.title("Validation")

def print_accuracies(input_file):
    """
    Print accuracy and balanced accuracy

    Arguments:
    input_file (str) -- Path to .csv that has the predictions with columns and Prediction, True_Label, Split
    Returns:
    accuracy and balanced accuracy
    """

    df = pd.read_csv(input_file)
    splits = ["training", "validation"]

    for split in splits:
        df_intermediate = df[df['split'] == split]
        print("The " + str(split) + " accuracy is: ", round(accuracy_score(df_intermediate.True_Label, df_intermediate.Prediction)*100,2), "%")
        print("The " + str(split) + " balanced accuracy is: ", round(balanced_accuracy_score(df_intermediate.True_Label, df_intermediate.Prediction)*100,2), "%")
        print()


from spec2test import TestSuite

def main():
    testsuite = TestSuite(input_path="./",
                          output_path="./",
                          units_=10,
                          learn_result_="./model_iter_72306")

    for _ in range(10):
        testsuite.generate("パソコン")

if __name__ == '__main__':
    main()