#!/usr/bin/python3
# -*- coding: utf-8 -*-
"""
======================
Laplacian segmentation
======================

This notebook implements the laplacian segmentation method of
`McFee and Ellis, 2014 <http://bmcfee.github.io/papers/ismir2014_spectral.pdf>`_,
with a couple of minor stability improvements.
This implementation is available at https://librosa.github.io/librosa/auto_examples/plot_segmentation.html

Additional functions have been added to the core segmentation:
 - unsupervised determination of the number of clusters suitable for the running task
 - different feature packages: spectral, cepstral and chroma.
 - a cosine distance between the different clusters that is plot together with cluster segmentation
 - a set of parameters reported in params.py file necessary for tuning the segmentation model.

usage:
python3 spectral_clustering_audio.py audiofilename.wav [.mp3]

Input:
 - name of audio file to be analyzed

Output:
 - Segmentation and grouping of the different musical sections synchronized on user-chosen onsets
 - Optional plots of similarity and recurrence matrix
 - Optional timestamps text file with parameters and time boundaries
"""

# Code source by Marie Tahon (2018) adapted from Brian McFee (2014)
# License: ISC

###################################
# Imports
#   - numpy for basic functionality
#   - scipy for graph Laplacian
#   - matplotlib for visualization
#   - sklearn.cluster for K-Means, for metrics and scaling.
#   - warnings to delete warning message for scipy package

from __future__ import division
import numpy as np
import scipy
import warnings
warnings.filterwarnings(action="ignore", module="scipy", message="^internal gelsd")
import sys, os
import argparse
import matplotlib.pyplot as plt
from matplotlib import gridspec

import sklearn.cluster
from sklearn.preprocessing import scale
import sklearn.metrics
import sklearn.utils

import librosa
import librosa.display

import cluster_rotate
import params

plt.rcParams.update({'font.size': 8})

BINS_PER_OCTAVE = params.BINS_PER_OCTAVE
N_OCTAVES = params.N_OCTAVES
NFFT = int(params.NFFT)
STEP = int(params.STEP)

#######################################
def detect_onsets(y, sr, M):
	#detect onsets
	oenv = librosa.onset.onset_strength(S=M, sr=sr)
	# Detect events without backtracking
	onset_raw = librosa.onset.onset_detect(onset_envelope=oenv, backtrack=False)
	## Backtrack the events using the onset envelope
	onset_bt = librosa.onset.onset_backtrack(onset_raw, oenv)
	# we fix_frames to include non-beat frames 0 and C.shape[1] (final frame)
	onset_frames = librosa.util.fix_frames(onset_raw, x_min=0, x_max=M.shape[1]-1)
	onset_times = librosa.frames_to_time(onset_frames, sr=sr, hop_length = STEP)
	# To reduce dimensionality, we'll beat-synchronous the CQT
	Msync = librosa.util.sync(M, onset_raw, aggregate=np.median)

	if params.onset_plot:
		plt.figure(figsize=(12, 4))
		plt.plot(oenv, label='Onset strength')
		plt.vlines(onset_raw, 0, oenv.max(), label='Raw onsets')
		plt.vlines(onset_bt, 0, oenv.max(), label='Backtracked', color='r')
		plt.legend(frameon=True, framealpha=0.75)
		plt.tight_layout()

		plt.figure(figsize=(12, 4))
		plt.subplot(2,1,1)
		plt.title('CQT spectrogram')
		librosa.display.specshow(M, y_axis='cqt_hz', sr=sr, hop_length= STEP, bins_per_octave=BINS_PER_OCTAVE, x_axis='time')
		plt.tight_layout()

		plt.subplot(2,1,2)
		plt.title('CQT spectrogram synchronized on onsets')
		librosa.display.specshow(Msync, bins_per_octave=BINS_PER_OCTAVE, y_axis='cqt_hz', x_axis='time', x_coords=onset_times)
		plt.tight_layout()

	return onset_raw, onset_times, Msync

##############################################
def detect_beats(y, sr, M):
	tempo, beats = librosa.beat.beat_track(y=y, sr=sr, hop_length = STEP, trim=False)
	print('Detected tempo:  {0:.2f} bpm'.format(tempo))
	beat_period = np.diff(librosa.frames_to_time(beats, sr=sr, hop_length= STEP))
	print('mean beat period: {0:.2f} ; std beat period: {1:.2f}'.format(60/np.mean(beat_period), np.std(beat_period)))

	beats_frames = librosa.util.fix_frames(beats, x_min=0, x_max=M.shape[1]-1)
	beat_times = librosa.frames_to_time(beats_frames, sr=sr, hop_length = STEP)

	Msync = librosa.util.sync(M, beats_frames, aggregate=np.median)
	if params.onset_plot:
		plt.figure(figsize=(12, 4))
		plt.subplot(2,1,1)
		plt.title('CQT spectrogram')
		librosa.display.specshow(M, y_axis='cqt_hz', sr=sr, hop_length=STEP, bins_per_octave=BINS_PER_OCTAVE, x_axis='time')
		plt.tight_layout()
		# For plotting purposes, we'll need the timing of the beats
		# we fix_frames to include non-beat frames 0 and C.shape[1] (final frame)

		plt.subplot(2,1,2)
		plt.title('CQT spectrogram synchronized on beats')
		librosa.display.specshow(Msync, bins_per_octave=BINS_PER_OCTAVE, y_axis='cqt_hz', x_axis='time', x_coords=beat_times)
		plt.tight_layout()
	return beats_frames, beat_times, Msync

##############################################
def no_onsets(sr, M):

	onsets = np.arange(0, M.shape[1])
	onset_times = librosa.samples_to_time(onsets, sr=sr/STEP)

	if params.onset_plot:
		plt.figure(figsize=(12, 4))
		plt.title('CQT spectrogram')
		librosa.display.specshow(M, y_axis='cqt_hz', sr=sr, bins_per_octave=BINS_PER_OCTAVE, x_axis='time', x_coords=onset_times)
		plt.tight_layout()

	return onsets, onset_times, M

def get_manual_beats(sr, M, filename):
	with open(filename, 'r') as f:
		data = f.readlines()
	times = np.array([float(x.strip()) for x in data[1:]])
	frames = np.array([int(x * sr / STEP) for x in times])
	onsets = librosa.util.fix_frames(frames, x_min=0, x_max=M.shape[1]-1)
	onset_times = librosa.frames_to_time(onsets, sr=sr, hop_length = STEP)

	Msync = librosa.util.sync(M, onsets, aggregate=np.median)

	if params.onset_plot:
		plt.figure(figsize=(12, 4))
		plt.subplot(2,1,1)
		plt.title('CQT spectrogram')
		librosa.display.specshow(M, y_axis='cqt_hz', sr=sr, hop_length=STEP, bins_per_octave=BINS_PER_OCTAVE, x_axis='time')
		plt.tight_layout()

		plt.subplot(2,1,2)
		plt.title('CQT spectrogram synchronized on beats')
		librosa.display.specshow(Msync, bins_per_octave=BINS_PER_OCTAVE, y_axis='cqt_hz', x_axis='time', x_coords=onset_times)
		plt.tight_layout()

	return onsets, onset_times, Msync

def extract_onsets(y, sr, manual_opt):
	method = params.onset
	#compute the CQT transform C: np.array((252, Tmax*sr/STEP))
	C = librosa.amplitude_to_db(librosa.core.magphase(librosa.cqt(y=y, sr=sr, bins_per_octave=BINS_PER_OCTAVE, n_bins=N_OCTAVES * BINS_PER_OCTAVE, hop_length = STEP))[0], ref=np.max)
	#to reduce dimensionality, we'll onset-synchronous the CQT
	#onset is a vector of onset indexes np.array((N+1,)) including 0
	#onset_times is a vector of onset times np.array((N+1,)) including 0
	#Csync is the CQT transform synchronized on onsets np.array((252, N))
	if method == 'no':
		onset, onset_times, Csync = no_onsets(sr, C)
	elif method == 'onset':
		onset, onset_times, Csync = detect_onsets(y, sr, C)
	elif method == 'beat':
		onset, onset_times, Csync = detect_beats(y, sr, C)
	elif method == 'manual':
		onset, onset_times, Csync = get_manual_beats(sr, C, manual_opt)
	else:
		print('onset parameter is not well-defined')
		sys.exit()

	return onset, onset_times, Csync

def build_weighted_rec_matrix(M):
	# Let's build a weighted recurrence affinity matrix using onset-synchronous CQT

	# the similarity matrix is filtered to prevent linkage errors and fill the gaps
	# the filter corresponds to a width=3 time window and a majority vote.
	R = librosa.segment.recurrence_matrix(M, width=3, mode='affinity',sym=True)

	# Enhance diagonals with a median filter
	df = librosa.segment.timelag_filter(scipy.ndimage.median_filter)
	Rf = df(R, size=(1, 7))
	return Rf

def build_seq_matrix(M, x):
	#build the sequence matrix using feature-similarity
	#Rpath[i, i+/-1] = \exp(- |M[i] - C[i+/-1]|^2 / sigma^2)`

	#synchronize features with onsets
	Msync = librosa.util.sync(M, x, aggregate=np.median)
	#Msync = M #pas de syncrhonisation

	#normalize (rescale) features between 0 and 1
	Msync_normed = scale(Msync)

	#constant scaling
	path_distance = np.sum(np.diff(Msync_normed, axis=1)**2, axis=0)
	#sigma is the median distance between successive beats/onsets.
	sigma = np.median(path_distance)
	path_sim = np.exp(-path_distance / sigma)

	#local scaling from A Spectral Clustering Approach to Speaker Diarization, Huazhong Ning, Ming Liu, Hao Tang, Thomas Huang
	R_path = np.diag(path_sim, k=1) + np.diag(path_sim, k=-1)
	return R_path

def build_laplacian_and_evec(Rf, R_path, opt, onsets):

	# And compute the balanced combination A of the two similarity matrices Rf and R_path
	deg_path = np.sum(R_path, axis=1)
	deg_rec = np.sum(Rf, axis=1)
	mu = deg_path.dot(deg_path + deg_rec) / np.sum((deg_path + deg_rec)**2)
	print('Optimal weight value (mu): {0:.2f}'.format(mu))

	A = mu * Rf + (1 - mu) * R_path

	# Plot the resulting graphs
	if opt: plot_similarity(Rf, R_path, A, onsets)

	# L: symetrized normalized Laplacian
	L = scipy.sparse.csgraph.laplacian(A, normed=True)

	# and its spectral decomposition (Find eigenvalues w and optionally eigenvectors v of matrix L)
	evals, evecs = np.linalg.eigh(L)
	print('L shape:', L.shape)

	# We can clean this up further with a median filter.
	# This can help smooth over small discontinuities
	evecs = scipy.ndimage.median_filter(evecs, size=(9, 1))

	# cumulative normalization is needed for symmetric normalize laplacian eigenvectors
	Cnorm = np.cumsum(evecs**2, axis=1)**0.5

	return Cnorm, evals, evecs

################################################
def compute_nb_clusters(method, evals, evecs, Tmax):

	if method == 'fixed':
		c = params.cluster_nb # list
	elif method == 'max':
		nc = []
		for it in range(params.cluster_max):
			nc.append(cluster_rotate.cluster_rotate(evecs/Cnorm, evals, range(1,10), 1, False))
		c = [int(np.mean(nc))+1]
	elif method == 'evals':
		ind = np.where(1- evals > 0.75)[0]
		#print(ind)
		return [len(ind)+1 ]
	elif method in ['silhouette', 'davies_bouldin', 'calinski_harabaz']:
		list_k = range(2,50,2)
		Cnorm = np.cumsum(e**2, axis=1)**0.5 #eigenvectors in input
		for k in list_k:
			print('nb of clusters:', k)
			X = e[:, :k] / Cnorm[:, k-1:k]
			# Let's use these k components to cluster beats into segments
			# (Algorithm 1)
			KM = sklearn.cluster.KMeans(n_clusters=k)
			seg_ids = KM.fit_predict(X)
			score = []
			if method == 'silhouette':
				score.append(sklearn.metrics.silhouette_score(X, seg_ids, metric='euclidean')) #max (proche de 1)
			elif method == 'davies_bouldin':
				score.append(davies_bouldin_score(X, seg_ids)) #min
			elif method == 'calinski_harabaz':
				score.append(sklearn.metrics.calinski_harabaz_score(X, seg_ids)) #max

		if method == 'silhouette':
			return list_k[np.argmax(score)]
		elif method == 'davies_bouldin':
			return list_k[np.argmin(score)]
		elif method == 'calinski_harabaz':
			return list_k[np.argmax(score)]

	else:
		print('method for finding the right number of clusters is unknown')
		sys.exit()

	print('nb of clusters:', c)

	return c

def davies_bouldin_score(X, labels):
	"""Computes the Davies-Bouldin score.
	The score is defined as the ratio of within-cluster distances to
	between-cluster distances.
	Read more in the :ref:`User Guide <davies-bouldin_index>`.
	Parameters
	----------
	X : array-like, shape (``n_samples``, ``n_features``)
	List of ``n_features``-dimensional data points. Each row corresponds
	to a single data point.
	labels : array-like, shape (``n_samples``,)
	Predicted labels for each sample.
	Returns
	-------
	score: float
	The resulting Davies-Bouldin score.
	References
	----------
	.. [1] `Davies, David L.; Bouldin, Donald W. (1979).
	"A Cluster Separation Measure". IEEE Transactions on
	Pattern Analysis and Machine Intelligence. PAMI-1 (2): 224-227`_
	"""
	X, labels = sklearn.utils.check_X_y(X, labels)
	le = sklearn.preprocessing.LabelEncoder()
	labels = le.fit_transform(labels)
	n_samples, _ = X.shape
	n_labels = len(le.classes_)
	if not 1 < n_labels < n_samples:
		raise ValueError("Number of labels is %d. Valid values are 2 to n_samples - 1 (inclusive)" % n_labels)

	intra_dists = np.zeros(n_labels)
	centroids = np.zeros((n_labels, len(X[0])), dtype=np.float)
	for k in range(n_labels):
		cluster_k = sklearn.utils.safe_indexing(X, labels == k)
		centroid = cluster_k.mean(axis=0)
		centroids[k] = centroid
		intra_dists[k] = np.average(sklearn.metrics.pairwise.pairwise_distances(cluster_k, [centroid]))

	centroid_distances = sklearn.metrics.pairwise.pairwise_distances(centroids)

	if np.allclose(intra_dists, 0) or np.allclose(centroid_distances, 0):
		return 0.0

	score = (intra_dists[:, None] + intra_dists) / centroid_distances
	score[score == np.inf] = np.nan

	return np.mean(np.nanmax(score, axis=1))

def plot_similarity(Rf, R_path, A, onset_times):

	plt.figure(figsize=(12, 4))
	plt.subplot(1, 3, 1)
	librosa.display.specshow(Rf, cmap='inferno_r', y_axis='time', y_coords=onset_times)
	plt.title('Long-range recurrence similarity (Rrec)')
	plt.subplot(1, 3, 2)
	librosa.display.specshow(R_path, cmap='inferno_r')
	plt.title('Local path similarity (Rloc)')
	plt.subplot(1, 3, 3)
	librosa.display.specshow(A, cmap='inferno_r')
	plt.title('Combined graph (A = m Rrec + (1-m) Rloc)')
	plt.tight_layout()

def plot_structure(Rf, X, seg_ids, k, onset_times):

	fig_s = plt.figure(figsize=(12, 4))
	colors = plt.get_cmap('Paired', k)

	ax_s1 = fig_s.add_subplot(1, 3, 2)
	librosa.display.specshow(Rf, cmap='inferno_r')
	ax_s1.set_title('Long-range recurrence similarity (Rrec)')
	ax_s2 =fig_s.add_subplot(1, 3, 1)
	librosa.display.specshow(X, y_axis='time', y_coords=onset_times)
	ax_s2.set_title('Structure components (Eigen vectors)')
	ax_s3 = fig_s.add_subplot(1, 3, 3)
	librosa.display.specshow(np.atleast_2d(seg_ids).T, cmap=colors)
	ax_s3.set_title('Estimated segments')
	plt.colorbar(ticks=range(k))
	plt.tight_layout()

#################################################
def compute_musical_density(C, onset_times, w, alpha):
	N = C.shape[1]
	density = []
	for n in range(N):
		t1 = np.min([onset_times[-1], onset_times[n] + w])
		t2 = np.min([onset_times[-1] -w, onset_times[n]])
		idw = np.where((onset_times < t1) & (onset_times >= t2))
		#if n + w < :
		threshold_chroma = np.max(C[:,idw])
		#else:
			#threshold_chroma = np.mean(C[:, N - w : N])
		idx = np.where(C[:,n] > alpha * threshold_chroma)
		density.append(len(idx[0]))

	return density

def plot_features(X, onsets, onset_times):

	Xsync = librosa.util.sync(X, onsets, aggregate=np.median)

	#print(X.shape, Xsync.shape)
	#print(onset_times)

	if params.feat[0] == 'chroma':
		fig_c = plt.figure(figsize=(12, 6))
		ax0_c = fig_c.add_subplot(3,1,1)
		ax0_c.set_title('onset-synchronous chroma (12)')
		#ax0_c.pcolor(distance, cmap = 'plasma')
		librosa.display.specshow(Xsync[:12,:], y_axis='chroma', x_axis='time', x_coords=onset_times, cmap = 'OrRd')
		#plt.colorbar()

		ax1_c = fig_c.add_subplot(3,1,2, sharex = ax0_c)
		ax1_c.set_title('onset-synchronous delta chroma (12)')
		librosa.display.specshow(np.abs(Xsync[12:,:]), y_axis='chroma', x_axis='time', x_coords=onset_times, cmap = 'OrRd')
		#plt.colorbar()

		density = compute_musical_density(Xsync[:12,:], onset_times, params.norm_density_win, params.alpha)
		print(len(onset_times), len(density))
		ax2_c = fig_c.add_subplot(3,1,3, sharex = ax0_c)
		ax2_c.set_title('musical density')
		ax2_c.plot(onset_times, density)
		plt.tight_layout()

	elif params.feat[0] == 'cepstral':
		fig_s = plt.figure(figsize=(12, 6))
		ax0_s = fig_s.add_subplot(3,1,1)
		ax0_s.set_title('onset-synchronous MFCC (20)')
		librosa.display.specshow(Xsync[:21,:], x_axis='time', x_coords=onset_times)
		#plt.colorbar()
		#plt.tight_layout()

		ax1_s = fig_s.add_subplot(3,1,2, sharex = ax0_s)
		ax1_s.set_title('onset-synchronous delta MFCC (20)')
		librosa.display.specshow(np.abs(Xsync[20:,:]), x_axis='time', x_coords=onset_times)
		#plt.colorbar()

		density = compute_musical_density(Xsync[:21,:], onset_times, params.norm_density_win, params.alpha)
		ax2_s = fig_s.add_subplot(3,1,2, sharex = ax0_s)
		ax2_s.set_title('musical density')
		ax2_s.plot(onset_times, density)
		plt.tight_layout()
	else:
		print('these parameters can not be plot')

def load_wav_percu(filename, start, duration, opt_percussive_part):
	y, sr = librosa.load(filename, offset=start, duration = duration)

	if opt_percussive_part:
	#separate harmonics and percussives into two wavforms
		y_harmo, yo = librosa.effects.hpss(y)
		librosa.output.write_wav(filename + '_harmo.wav', y_harmo, sr)
		librosa.output.write_wav(filename + '_percu.wav', y_percu, sr)
		return yo, sr
	else:
		return y, sr

################################################
def feature_extraction(y, sr, opt_tuning):

	if opt_tuning:
		#extraction of tuning
		A440 = librosa.estimate_tuning(y=y, sr=sr, resolution=1e-3)
		print('Deviation from A440 is : {0:.2f}'.format(A440))
	else:
		A440 = 0.0

	print('Features for local similarity: ', ' '.join(params.feat))
	full = []
	idx_chroma = 0

	if 'cepstral' in params.feat:
		mfcc = librosa.feature.mfcc(y=y, sr=sr, n_mfcc = 20, n_fft = NFFT, hop_length = STEP)
		mfcc_delta = librosa.feature.delta(mfcc)
		fcep = np.concatenate((mfcc, mfcc_delta), axis=0)
		full.append(fcep)

	if 'chroma' in params.feat:
		chroma = librosa.feature.chroma_cqt(y=y, sr=sr, n_chroma = 12, n_octaves = N_OCTAVES, hop_length = STEP, norm = None, tuning= A440)
		chroma_delta = librosa.feature.delta(chroma)
		fchr = np.concatenate((chroma, chroma_delta), axis=0)
		idx_chroma = len(full)
		full.append(fchr)

	if 'spectral' in params.feat:
		centroid = librosa.feature.spectral_centroid(y=y, sr=sr, n_fft = NFFT, hop_length = STEP)
		contrast = librosa.feature.spectral_contrast(y=y, sr=sr, n_fft = NFFT, n_bands=6, hop_length = STEP)
		flatness = librosa.feature.spectral_flatness(y=y, n_fft = NFFT, hop_length = STEP)
		rolloff05 = librosa.feature.spectral_rolloff(y=y, sr= sr, n_fft = NFFT, hop_length = STEP, roll_percent= 0.05)
		rolloff25 = librosa.feature.spectral_rolloff(y=y, sr= sr, n_fft = NFFT, hop_length = STEP, roll_percent= 0.25)
		rolloff50 = librosa.feature.spectral_rolloff(y=y, sr= sr, n_fft = NFFT, hop_length = STEP, roll_percent= 0.50)
		rolloff75 = librosa.feature.spectral_rolloff(y=y, sr= sr, n_fft = NFFT, hop_length = STEP, roll_percent= 0.75)
		rolloff95 = librosa.feature.spectral_rolloff(y=y, sr= sr, n_fft = NFFT, hop_length = STEP, roll_percent= 0.95)
		spec = np.concatenate((centroid, contrast, flatness, rolloff05,rolloff25,rolloff50,rolloff75,rolloff95), axis=0)
		spec_delta = librosa.feature.delta(spec)
		fspec = np.concatenate((spec, spec_delta), axis = 0)
		full.append(fspec)

	full = np.array(full)[0]

	print('feature shape', full.shape)
	return full, idx_chroma

def extract_time_boundaries(cluster_ids, onsets, nb_frames, sr):

	# Locate segment boundaries from the label sequence
	bound_beats = 1 + np.flatnonzero(cluster_ids[:-1] != cluster_ids[1:])

	# Count beat 0 as a boundary
	bound_beats = librosa.util.fix_frames(bound_beats, x_min=0)

	# Compute the segment label for each boundary
	bound_labels = list(cluster_ids[bound_beats])

	# Convert beat indices to frames
	bound_frames = onsets[bound_beats]

	# Make sure we cover to the end of the track
	bound_frames = librosa.util.fix_frames(bound_frames,  x_min=None, x_max=nb_frames-1)
	bound_times = librosa.frames_to_time(bound_frames, sr=sr, hop_length = STEP)

	return bound_times, bound_labels

##################################
def extract_cosine_distance_clusters(center_clusters, distance_ref, type_dist = 'cos'):
	distance = []

	for center in center_clusters:
		if type_dist == 'cos':
			distance.append( scipy.spatial.distance.cosine( center, distance_ref) )
		elif type_dist == 'eucl':
			distance.append(np.sqrt( np.sum( (center - distance_ref)**2) ))

	return distance

def extract_distance_between_clusters(center_clusters, type_dist = 'cos'):
	distance = np.zeros((center_clusters.shape))

	for i, center_i in enumerate(center_clusters):
		for j, center_j in enumerate(center_clusters):
			if type_dist == 'cos':
				distance[i,j] = scipy.spatial.distance.cosine( center_i, center_j)
			elif type_dist == 'eucl':
				distance[i,j] = np.sqrt( np.sum( (center_i - center_j)**2) )

	x = range(i+1)
	y = range(j+1)
	xloc = [c + 0.5 for c in x]
	cx = [str(c) for c in x]
	#print(cx)
	fig_d, ax_d = plt.subplots(figsize=(5, 4))
	p_d = ax_d.pcolor(distance, cmap = 'inferno_r')
	cb = fig_d.colorbar(p_d)
	ax_d.xaxis.set_ticks(xloc)
	ax_d.xaxis.set_ticklabels(cx)
	ax_d.yaxis.set_ticks(xloc)
	ax_d.yaxis.set_ticklabels(cx)
	ax_d.set_title('Distance between clusters')
	ax_d.set_xlabel('clusters numbers')
	plt.tight_layout()

	return distance

def extract_ref_signal(X, onset_times):
	ind = np.where((onset_times >= params.begin_ref) & (onset_times < params.end_ref))
	return X[ind,:]

def main():

	parser = argparse.ArgumentParser(description='Segmentation and clustering of musical sections with spectral clustering (Laplacian matrix and eigen values)')
	parser.add_argument('filename', type=str, help='name of audio file')
	parser.add_argument('manual_onset', nargs='?', type=str, help='name of the file containing manual annotations for onset timestamps (with method=manual)')

	args = parser.parse_args()

	#==================
	# Signal processing
	#==================

	#extract waveform from audio signal of given duration and begining. If onset_percu is True, extract only percussive part of the signal.
	y, sr = load_wav_percu(args.filename, params.begin, params.duration, params.onset_percu)
	print('signal shape:', y.shape, ' sr=', sr, 'win duration=%.2f' %(NFFT / sr))

	#extract acoustic feature from audio signal feat is a matrix np.array((nb features, Tmax*sr/STEP))
	feat, idx_chroma = feature_extraction(y, sr, params.opt_tuning)

	#extract onset indexes and times + onset-synchronous CQT transform on onsets.
	onsets, onset_times, Csync = extract_onsets(y, sr, args.manual_onset)

	#if 'chroma' in params.feat:
	#	compute_musical_density(Csync, onset_times, idx_chroma, params.norm_density_win, params.alpha, sr)

	if params.plot_features: plot_features(feat, onsets, onset_times)

	#================
	# Affinity matrix
	#================

	#compute a non-negative affinity matrix using onset-synchronous CQT (with Gaussian kernel)
	#represent local consistency of timbral (CQT) features
	Rf = build_weighted_rec_matrix(Csync)

	#compute a non-negative affinity matrix using onset-synchronous feature matrix (with Gaussian kernel)
	#represent long-range repeating forms of harmonic features
	R_path = build_seq_matrix(feat, onsets)

	#compute Laplacian (sequence augmented affinity matrix) as a linear combination of Rf and Rpath and extract eigenvalues and vectors.
	Cnorm, evals, evecs = build_laplacian_and_evec(Rf, R_path, params.plot_simi, onset_times)

	#===========
	# Clustering
	#===========

	#determine number of clusters kl is a list of potential numbers of cluster.
	kl = compute_nb_clusters(params.cluster_method, evals, evecs, y.shape[0]*sr)
	N_CLUST = len(kl)

	#=================
	# Start plotting
	#=================
	import matplotlib.patches as patches
	fig_f = plt.figure(figsize = (12, 3+2*N_CLUST))
	#fig.subplots_adjust(hspace=.5)

	#plot onset-synchronous CQT
	hr = [1] * (N_CLUST +1)
	hr[0] = 2
	gs = gridspec.GridSpec(1 + N_CLUST,1, height_ratios=hr)
	ax_f0 = fig_f.add_subplot(gs[0])
	librosa.display.specshow(Csync, y_axis='cqt_hz', sr=sr, hop_length = STEP, bins_per_octave=BINS_PER_OCTAVE, x_axis='time', x_coords=onset_times)
	#librosa.display.specshow(feat, y_axis='chroma', x_axis='time') #ou
	ax_f0.set_title('CQT spectrogram synchronized {0}'.format(params.onset))

	for it, k in enumerate(kl):
		#limit the number of clusters per second
		if k > params.cluster_nb_max*sr*y.shape[0]:
			k = params.cluster_nb_max*sr*y.shape[0]
		print('nb of clusters: {} for it {}/{}'.format(k, it, N_CLUST))

		#for k clusters, use the first k normalized eigenvectors.
		#X can be interpretable as an onset-synchronous matrix containing relevant feature information for local and log-range structure segmentation
		X = evecs[:, :k] / Cnorm[:, k-1:k]

		#onsets are grouped into k clusters, each cluster having its own acoustic characteristics
		KM = sklearn.cluster.KMeans(n_clusters=k)
		#seg_ids is a np.array((label)) label being a number corresponding to one cluster seg_ids[i] is the label of onset i
		seg_ids = KM.fit_predict(X)

		#if needed compute the cosine distance between each cluster and a reference taken at the very begining of th signal
		#KM.cluster_centers_ : array, [n_clusters, n_features]
		if params.cluster_dist:
			ref_signal = extract_ref_signal(X, onset_times)
			distance_cosine_cluster = extract_cosine_distance_clusters( KM.cluster_centers_, np.mean(X[:10*NFFT,:], axis=0))
		else:
			distance_cosine_cluster = None

		if params.plot_dist:
			distance_between_clusters = extract_distance_between_clusters( KM.cluster_centers_ )

		# and plot the resulting structure representation
		if params.plot_struct: plot_structure(Rf, X, seg_ids, k, onset_times)

		bound_times, bound_labels = extract_time_boundaries(seg_ids, onsets, feat.shape[1], sr)
		freqs = librosa.cqt_frequencies(n_bins=Csync.shape[0], fmin=librosa.note_to_hz('C1'), bins_per_octave=BINS_PER_OCTAVE)

		timestamps_name = os.path.splitext(args.filename)[0] + '_timestamps.txt'

		#=============
		# Plot results
		#=============

		cmap = plt.get_cmap('Paired', k)
		#write header of text file with parameters.
		if params.timestamps:
			f = open(timestamps_name, 'a')
			f.write('WIN = {0:.2f} sec, NFFT = {1}, STEP = {2}, begin = {3}, duration = {4}\n'.format(NFFT / sr, NFFT, STEP, params.begin, params.duration))
			f.write('Nb of clusters: {0} obtained with method {1} and features {2}\n'.format(k, params.cluster_method, '-'.join(params.feat)))

		#plot onset-synchronous CQT
		#if it == 0:

		#plot segmentation and clusters grouping (+ cosine distance.)
		#also write obtained boundaries in the text file.
		ax_f1 = fig_f.add_subplot(gs[it + 1], sharex = ax_f0)
		for interval, label in zip(zip(bound_times, bound_times[1:]), bound_labels):
			if params.timestamps: f.write('{0:.2f} \t {1:.2f} \t {2} \n'.format(interval[0], interval[1], label))
			if params.cluster_dist: ax_f1.plot([interval[0], interval[1]],[distance_cosine_cluster[label], distance_cosine_cluster[label]], 'k')
			ax_f1.add_patch(patches.Rectangle((interval[0], 0), interval[1] - interval[0], 1, facecolor=cmap(label), alpha=1))
			ax_f1.text(interval[0]+(interval[1]-interval[0])/2, 0.9, label, fontsize=8)
		if params.timestamps: f.close()

		#plt.subplots_adjust(hspace=.0)
	plt.tight_layout()
	plt.show()

if __name__ == '__main__':
	main()

title = 'Palestrina'
# Palestrina, AccordsMajeurs, AccordsMineur, Majeur3et4notes, Majeur3et4notes, Accords3Notes, DispoMajeurMineur, Tension
# Cadence3V, Cadence4VMaj, Cadence4Vmin,
audio = load('/Users/manuel/Dropbox (TMG)/Thèse/code/DescripteursHarmoniquesAudio/'+title+'.wav')
main(audio)


# 5_chart_1.py
import streamlit as st
import pandas as pd
import plotly.express as px

# 데이터 불러오기
data = 'data/gdp_by_country.csv'
df = pd.read_csv(data)

# 원본 데이터에서 국가명 리스트 가져오기
clist = df['country'].unique()

# 사이드바에서 국가명 선택
country = st.sidebar.selectbox("국가를 선택하세요:",clist)

# 메인화면
st.subheader('GDP per Capita over time')

# 선택한 국가명에 따라 연도별 gdp 추이를 라인 그래프 설정하기
fig = px.line(df[df['country'] == country],
    x = "year", y = "gdpPercap", title = country)

# plotly 그래프 출력하기
st.plotly_chart(fig)

# 원본 데이터에서 선택한 국가의 rawdata 확인하기
st.subheader('data')
st.write(df[df['country'] == country])


from itertools import cycle

with open("Data\Day03.txt") as file:
    input = [line.strip() for line in file]

def getitem(index, iterable):
    for idx, val in enumerate(iterable):
        if idx == index:
            return val

def istree(row, rowidx, rowStep, colStep):
    if rowidx % rowStep != 0:
        return False
    colidx = rowidx * colStep
    element = getitem(colidx, cycle(row))
    return element == '#'

def solve_part1(items):
    count = 0
    for rowidx, row in enumerate(input):
        count = count + 1 if istree(row, rowidx, 1, 3) else count
    return count

def solve_part2(items):
    steps = [(1, 1), (1, 3), (1, 5), (1, 7), (2, 0.5)]
    count = [0] * len(steps)
    for rowidx, row in enumerate(input):
        for idx, (rowstep, colstep) in enumerate(steps):
            count[idx] = count[idx] + 1 if istree(row, rowidx, rowstep, colstep) else count[idx]
    return count[0] * count[1] * count[2] *  count[3] * count[4]

print(f"Part 1: {solve_part1(input)}")
print(f"Part 2: {solve_part2(input)}")


from flask import Flask, Markup, render_template
from datetime import timezone, datetime
import database

db = database.connect()
app = Flask(__name__)

@app.route("/")
def hello_world():
    (temp_values,temp_labels) = get_temp_data()
    (fan_values,fan_labels) = get_fan_data()
    return render_template('line_chart.html', title='All the info', max=30, labels=temp_labels, values=temp_values, max2=1, labels2=fan_labels, values2=fan_values)

def get_temp_data():
    temp_data = database.select(db, "select temperature, timestamp from temperature_log where timestamp > datetime('now','-2 day','localtime') order by timestamp asc")
    values = []
    labels = []
    for row in temp_data:
        process_row(row, values, labels)
    return (values,labels)

def get_fan_data():
    temp_data = database.select(db, "select power, timestamp from fan_log where timestamp > datetime('now','-2 day','localtime') order by timestamp asc")
    values = []
    labels = []
    for row in temp_data:
        process_row(row, values, labels)
    return (values,labels)

def process_row(row, values, labels):
    val, timestamp = row
    timestamp = datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S")
    timestamp_new = (timestamp - datetime(1970, 1, 1)).total_seconds()
    values.append(val)
    labels.append(timestamp)


import collections
import time

from .neo import neo
from .options import options

__all__ = ['logger']

class __Logger:

    def create_reset_log_history(self):
        self.max_history = options.get('max_log_history', cache=False) or None
        self.log_history = collections.deque(maxlen=self.max_history)
        # Delete and wipe buffer from memory if it exists
        if neo.vim.call('bufexists', 'bugg_buffer_log_history'):
            neo.vim.command('bwipeout bugg_buffer_log_history')
        # Create new buffer to show log history
        self.buffer_log_history = neo.vim.call('nvim_create_buf', 0, 1)
        # Set the name of the log history buffer so we can find it
        neo.vim.call('nvim_buf_set_name',
            self.buffer_log_history, 'bugg_buffer_log_history'
        )

    @property
    def log_lines(self):
        return [
            '[{}] {}'.format(log['timestamp'], log['title'])
            for log in self.log_history
        ]

    def log(self, title, details=None):
        # Add log to python log queue
        timestamp = time.strftime('%H:%M:%S')
        self.log_history.appendleft({
            'title': title,
            'timestamp': timestamp,
            'details': details,
        })

        # Add log line to top of log history buffer
        log_line = '[{}] {}'.format(timestamp, title)
        neo.vim.call('nvim_buf_set_lines',
            self.buffer_log_history, 0, 0, 0, [log_line]
        )

        if self.max_history and (len(self.log_history) >= self.max_history):
            # Delete all lines above max_history
            neo.vim.call('nvim_buf_set_lines',
                self.buffer_log_history, self.max_history, -1, 0, []
            )

logger = __Logger()


"""
An example submitit script for running multiple jobs in parallel on a SLURM cluster.
"""
import datetime
import shlex
import sys
import time
from pathlib import Path

try:
    import submitit
    from tap import Tap
except Exception as e:
    print('Please install submitit with `pip install submitit`')
    print('Please install tap with `pip install typed-argument-parser`')
    raise(e)

class SubmitItArgs(Tap):
    partition: str = 'your-slurm-partition'
    submit: bool = False

# Args
args = SubmitItArgs().parse_args()

# Commands
commands = []
for category in ['hydrant', 'toytruck']:
    command = f"""python main.py dataset.category={category} dataloader.batch_size=24 dataloader.num_workers=8 run.vis_before_training=True run.val_before_training=True run.name=check_train__{category}__ebs_24"""
    commands.append(command)

# Create executor
Path("slurm_logs").mkdir(exist_ok=True)
executor = submitit.AutoExecutor(folder="slurm_logs")
executor.update_parameters(
    tasks_per_node=1,
    timeout_min=36 * 60,
    slurm_partition=args.partition,
    slurm_gres="gpu:1",
    slurm_constraint="volta32gb",
    slurm_job_name="submititjob",
    cpus_per_task=8,
    mem_gb=32.0,
)

# Check
if not args.submit:
    for command in commands:
        print(command)
    sys.exit()

# Submitit via SLURM array
print('Start')
print(datetime.datetime.now().strftime("%Y-%m-%d--%H-%M-%S"))
start_time = time.time()
jobs = []
with executor.batch():
    print('SUBMITTING:')
    for command in commands:
        function = submitit.helpers.CommandFunction(shlex.split(command), verbose=True)
        job = executor.submit(function)
        jobs.append(job)

# Then wait until all jobs are completed:
outputs = [job.result() for job in jobs]
print(f'Finished all ({len(outputs)}) jobs in {time.time() - start_time:.1f} seconds')
print(datetime.datetime.now().strftime("%Y-%m-%d--%H-%M-%S"))


from django.http import HttpResponse
from django.shortcuts import render, redirect
from django.contrib.auth.decorators import login_required
from base.models import UserProfile
from base.serializers import ProfileSerializer
from rest_framework.decorators import api_view
from rest_framework.response import Response
from .forms import UpdateProfileForm
from django.contrib import messages
from django.contrib.auth.forms import UserCreationForm

def index(request):
    return render(request, 'index.html')

@login_required
def profile(request):
    if request.method == 'POST':
        profile_form = UpdateProfileForm(request.POST, instance=request.user.profile)
        print(profile_form.is_valid())
        if profile_form.is_valid():
            profile_form.save()
            messages.success(request, 'Your profile is updated successfully')
            return redirect(to='index')
        else:
            print(profile_form.errors)
            # for i in profile_form.errors.items:
            #     print(i.value)
    else:
        profile_form = UpdateProfileForm(instance=request.user.profile)

    return render(request, 'registration/profile.html', {'profile_form': profile_form})

@api_view(['GET'])
def listUsers(request):
    profiles = UserProfile.objects.all()
    serializer = ProfileSerializer(profiles, many=True)
    return Response(serializer.data)

def register(request):
    if request.method == 'POST':
        form = UserCreationForm(request.POST)
        if form.is_valid():
            form.save()

            messages.success(request, f'Your account has been created. You can log in now!')
            return redirect('login')
    else:
        form = UserCreationForm()

    context = {'form': form}
    return render(request, 'registration/register.html', context)


from typing import List

# Approach 1 -> Slicing Method
def reverseList(inputList: List) -> List:
    if(len(inputList) <= 1):
        return inputList

    return inputList[:: -1]

# Approach 2 -> Inbuilt Method
def reverseList2(inputList: List) -> List:
    if(len(inputList) <= 1):
        return inputList

    inputList.reverse()
    return inputList

# Approach 3 -> RAW SWAPPING :)
def reverseList3(inputList: List) -> List:
    if(len(inputList) <= 1):
        return inputList

    lastIndex = len(inputList) - 1
    startIndex = 0
    while startIndex < lastIndex:
        inputList[startIndex], inputList[lastIndex] = inputList[lastIndex], inputList[startIndex]
        startIndex += 1
        lastIndex -= 1
    return inputList

print(reverseList(inputList=[10, 20, 30, 40, 19, 21]))
print(reverseList(inputList=[10]))
print(reverseList(inputList=[10, 20]))

print("\nMethod 2")
print(reverseList2(inputList=[10, 20, 30, 40, 19, 21]))
print(reverseList2(inputList=[10]))
print(reverseList2(inputList=[10, 20]))

print("\nMethod 3")
print(reverseList3(inputList=[10, 20, 30, 40, 19, 21]))
print(reverseList3(inputList=[10]))
print(reverseList3(inputList=[10, 20]))


import kivy
from kivy.app import App
from kivy.uix.label import Label
from kivy.uix.gridlayout import GridLayout
from kivy.uix.textinput import TextInput
from kivy.uix.button import Button
from kivy.uix.widget import Widget
from kivy.properties import ObjectProperty
from kivy.config import Config
from kivy.core.window import Window
from kivy.lang import Builder
from kivy.uix.screenmanager import ScreenManager, Screen
from kivy.uix.dropdown import DropDown
from kivy.uix.textinput import TextInput
from kivy.clock import Clock
from kivy.uix.popup import Popup
from kivy.uix.scrollview import ScrollView
import sqlite3

lijst_dagen = []
dagen_popup = []

#Verschillende schermen benoemen
class Navbar(Screen):
    pass
class Dashboard(Screen):
    pass
class Planning(Screen):
    pass
class Schoolwerk(Screen):
    def on_enter(self):
        pw_lijst = Scorro.show_proefwerken(self)
        self.ids.BoxHwPw.clear_widgets()
        for item in pw_lijst:
            replace = str(item).replace("(", "")
            replace = replace.replace(")", "")
            replace = replace.replace("'", "")
            button = Button(text=str(replace))
            self.ids.BoxHwPw.add_widget(button)

        hw_lijst = Scorro.show_huiswerk(self)
        for item in hw_lijst:
            replace = str(item).replace("(", "")
            replace = replace.replace(")", "")
            replace = replace.replace("'", "")
            button = Button(text=str(replace))
            self.ids.BoxHwPw.add_widget(button)

class PopupVak(Popup):
    def on_open(self):
        dagen_popup.clear()
        naam = self.title.split(" - ")[0]

        conn = sqlite3.connect('ScorroDB.db')
        c = conn.cursor()

        c.execute(f"SELECT * FROM vakken WHERE naam = '{naam}'")
        records = c.fetchall()

        conn.commit()
        conn.close()

        colour_selec = (0,1,0,1)
        colour_deselec = (1,0,0,1)

        try:
            dagen = records[0][1].replace("[", "").replace("]", "").replace(" ", "").replace("'", "").split(",")
            for dag in dagen:
                dagen_popup.append(dag)

            for dag in dagen_popup:
                self.ids[dag].background_color = colour_selec
        except:
            print("Geen dagen gevonden")

    def Savedag(self, dag):
        colour_selec = (0,1,0,1)
        colour_deselec = (1,0,0,1)
        if dag in dagen_popup:
            dagen_popup.remove(dag)
            self.ids[dag].background_color = colour_deselec
        else:
            dagen_popup.append(dag)
            self.ids[dag].background_color = colour_selec

    def OW_klas(self):
        text = self.ids.naam_vakAP.text
        old_name = self.title.split(" - ")[0]

        if dagen_popup != []:
            if text != "":
                conn = sqlite3.connect('ScorroDB.db')
                c = conn.cursor()

                dagen = str(dagen_popup)
                c.execute("""UPDATE vakken SET
                    naam = :naam,
                    dag = :dag
                    WHERE naam = :old_name""",
                    {
                    'naam': text,
                    'dag': str(dagen_popup),
                    'old_name': old_name,
                })

                conn.commit()
                conn.close()
                print("vak opgeslagen")

                dagen_popup.clear()

                vakken_screen = App.get_running_app().root.get_screen("vakken").ids.BoxVakken
                for vak in vakken_screen.children:
                    if vak.text == old_name:
                        vak.text = text

                sorted_buttons = sorted(vakken_screen.children, key=lambda x: x.text)
                vakken_screen.clear_widgets()
                for button in sorted_buttons:
                    vakken_screen.add_widget(button)
            else:
                print("Geen tekst")
        else:
            print("Geen dagen")

    def verwijderKlas(self):
        naam = self.title.split(" - ")[0]

        conn = sqlite3.connect('ScorroDB.db')
        c = conn.cursor()

        c.execute(f"DELETE FROM vakken WHERE naam = '{naam}'")

        conn.commit()
        conn.close()

        vakken_screen = App.get_running_app().root.get_screen("vakken").ids.BoxVakken

        for vak in vakken_screen.children:
            if vak.text == naam:
                vakken_screen.remove_widget(vak)

class Vakken(Screen):
    def popupVak(self, naam):
        popup = PopupVak(title=f"{naam} - vak aanpassen")
        popup.ids.naam_vakAP.text = naam
        popup.open()

    def on_enter(self):
        window_size = int(Window.size[1]) / 10
        vakken_lijst = Scorro.show_klassen(self)
        self.ids.BoxVakken.clear_widgets()

        for vak in vakken_lijst:
            button = Button(text=str(vak[0]), size_hint_y=None, height=window_size)
            button.bind(on_press=lambda button: self.popupVak(button.text))
            self.ids.BoxVakken.add_widget(button)

        sorted_buttons = sorted(self.ids.BoxVakken.children, key=lambda x: x.text)
        self.ids.BoxVakken.clear_widgets()
        for button in sorted_buttons:
            self.ids.BoxVakken.add_widget(button)

class Cijfers(Screen):
    def on_enter(self):
        print(Scorro.show_cijfers(self))

class NieuwHuiswerk(Screen):
    def spinnerHW_clicked(self):
        data = Scorro.show_klassen(self)
        spinner = self.ids.kiesvakHW
        spinner.values = [str(item[0]) for item in data]

class NieuwProefwerk(Screen):
    def spinnerPW_clicked(self):
        data = Scorro.show_klassen(self)
        spinner = self.ids.kiesvakPW
        spinner.values = [str(item[0]) for item in data]

class NieuwCijfer(Screen):
    def spinnerCF_clicked(self):
        data = Scorro.show_klassen(self)
        spinner = self.ids.kiesvakCF
        spinner.values = [str(item[0]) for item in data]

class NieuwVak(Screen):
    def on_enter(self):
        for item in lijst_dagen:
            lijst_dagen.remove(item)
        self.ids.maandag.background_color = (1,0,0,1)
        self.ids.dinsdag.background_color = (1,0,0,1)
        self.ids.woensdag.background_color = (1,0,0,1)
        self.ids.donderdag.background_color = (1,0,0,1)
        self.ids.vrijdag.background_color = (1,0,0,1)
        self.ids.zaterdag.background_color = (1,0,0,1)
        self.ids.zondag.background_color = (1,0,0,1)

    def hide_message(self):
        self.ids.BoxMessage.remove_widget(self.ids.BoxMessage.children[0])

    def SavedVak(self):
        text = self.ids.naam_vak.text
        if lijst_dagen != []:
            if text != "":
                message = Label(text="Vak opgeslagen!")
                self.ids.BoxMessage.add_widget(message)
                Clock.schedule_once(lambda x: self.hide_message(), 1.5)

    def Savedag(self, dag):
        colour_selec = (0,1,0,1)
        colour_deselec = (1,0,0,1)
        if dag in lijst_dagen:
            lijst_dagen.remove(dag)
            self.ids[dag].background_color = colour_deselec
        else:
            lijst_dagen.append(dag)
            self.ids[dag].background_color = colour_selec

class CijferBerekenen(Screen):
    pass

class WindowManager(ScreenManager):
    pass

class Scorro(App):
    def build(self):
        self.icon = "Images/Logo.png"

        conn = sqlite3.connect('ScorroDB.db')
        c = conn.cursor()

        c.execute("""CREATE TABLE if not exists vakken(
            naam text,
            dag text)
        """)

        c.execute("""CREATE TABLE if not exists cijfers(
            cijfer text,
            weging text,
            beschrijving text,
            vak text)
        """)

        c.execute("""CREATE TABLE if not exists proefwerken(
            naam text,
            datum text,
            beschrijving text,
            vak text)
        """)

        c.execute("""CREATE TABLE if not exists huiswerk(
            naam text,
            datum text,
            beschrijving text,
            vak text)
        """)

        conn.commit()
        conn.close()

        kv = Builder.load_file('testdl.kv')
        return kv

    #functies voor klassen
    def submit_klas(self):
        global lijst_dagen
        text = self.root.get_screen('nieuw vak').ids.naam_vak.text
        if lijst_dagen != []:
            if text != "":
                #connect to database
                conn = sqlite3.connect('ScorroDB.db')
                c = conn.cursor()

                c.execute("INSERT INTO vakken VALUES (:naam, :dag)",
                {
                    'naam': self.root.get_screen('nieuw vak').ids.naam_vak.text,
                    'dag': str(lijst_dagen),
                })

                conn.commit()
                conn.close()

                #reset buttons
                lijst_dagen = []
                self.root.get_screen('nieuw vak').ids.maandag.background_color = (1,0,0,1)
                self.root.get_screen('nieuw vak').ids.dinsdag.background_color = (1,0,0,1)
                self.root.get_screen('nieuw vak').ids.woensdag.background_color = (1,0,0,1)
                self.root.get_screen('nieuw vak').ids.donderdag.background_color = (1,0,0,1)
                self.root.get_screen('nieuw vak').ids.vrijdag.background_color = (1,0,0,1)
                self.root.get_screen('nieuw vak').ids.zaterdag.background_color = (1,0,0,1)
                self.root.get_screen('nieuw vak').ids.zondag.background_color = (1,0,0,1)

                self.root.get_screen('nieuw vak').ids.naam_vak.text = ''
                print("vak opgeslagen")

            else:
                print("Geen tekst")
        else:
            print("Geen dagen")

    def show_klassen(self):
        conn = sqlite3.connect('ScorroDB.db')
        c = conn.cursor()

        c.execute("SELECT * FROM vakken")
        records = c.fetchall()

        conn.commit()
        conn.close()

        return records

    #functies voor cijfers
    def submit_cijfer(self):
        naam = self.root.get_screen('nieuw cijfer').ids.welkCF.text
        weging = self.root.get_screen('nieuw cijfer').ids.wegingCF.text
        vak = self.root.get_screen('nieuw cijfer').ids.kiesvakCF.text
        if naam != "":
            if weging != "":
                if vak != "Selecteer een vak":
                    conn = sqlite3.connect('ScorroDB.db')
                    c = conn.cursor()

                    c.execute("INSERT INTO cijfers VALUES (:cijfer, :weging, :beschrijving, :vak)",
                    {
                        'cijfer': self.root.get_screen('nieuw cijfer').ids.welkCF.text,
                        'weging': self.root.get_screen('nieuw cijfer').ids.wegingCF.text,
                        'beschrijving': self.root.get_screen('nieuw cijfer').ids.infoCF.text,
                        'vak': self.root.get_screen('nieuw cijfer').ids.kiesvakCF.text,
                    })

                    self.root.get_screen('nieuw cijfer').ids.welkCF.text = ''
                    self.root.get_screen('nieuw cijfer').ids.wegingCF.text = ''
                    self.root.get_screen('nieuw cijfer').ids.infoCF.text = ''
                    self.root.get_screen('nieuw cijfer').ids.kiesvakCF.text = 'Selecteer een vak'

                    conn.commit()
                    conn.close()
                else:
                    print("Selecteer een vak")
            else:
                print("Geef je cijfer een weging")
        else:
            print("Geef je cijfer")

    def show_cijfers(self):
        conn = sqlite3.connect('ScorroDB.db')
        c = conn.cursor()

        c.execute("SELECT * FROM cijfers")
        records = c.fetchall()

        conn.commit()
        conn.close()
        return records

    #functies voor proefwerken
    def submit_proefwerk(self):
        naam = self.root.get_screen('nieuw proefwerk').ids.welkPW.text
        datum = self.root.get_screen('nieuw proefwerk').ids.datumPW.text
        vak = self.root.get_screen('nieuw proefwerk').ids.kiesvakPW.text
        if naam != "":
            if datum != "":
                if vak != "Selecteer een vak":
                    conn = sqlite3.connect('ScorroDB.db')
                    c = conn.cursor()

                    c.execute("INSERT INTO proefwerken VALUES (:naam, :datum, :beschrijving, :vak)",
                    {
                        'naam': self.root.get_screen('nieuw proefwerk').ids.welkPW.text,
                        'datum': self.root.get_screen('nieuw proefwerk').ids.datumPW.text,
                        'beschrijving': self.root.get_screen('nieuw proefwerk').ids.infoPW.text,
                        'vak': self.root.get_screen('nieuw proefwerk').ids.kiesvakPW.text,
                    })

                    self.root.get_screen('nieuw proefwerk').ids.welkPW.text = ''
                    self.root.get_screen('nieuw proefwerk').ids.datumPW.text = ''
                    self.root.get_screen('nieuw proefwerk').ids.infoPW.text = ''
                    self.root.get_screen('nieuw proefwerk').ids.kiesvakPW.text = 'Selecteer een vak'

                    conn.commit()
                    conn.close()
                else:
                    print("Selecteer een vak")
            else:
                print("Geef je proefwerk een datum")
        else:
            print("Geef je proefwerk een naam")

    def show_proefwerken(self):
        conn = sqlite3.connect('ScorroDB.db')
        c = conn.cursor()

        c.execute("SELECT * FROM proefwerken")
        records = c.fetchall()

        conn.commit()
        conn.close()
        return records

    #functies voor huiswerk
    def submit_huiswerk(self):
        naam = self.root.get_screen('nieuw huiswerk').ids.welkHW.text
        datum = self.root.get_screen('nieuw huiswerk').ids.datumHW.text
        vak = self.root.get_screen('nieuw huiswerk').ids.kiesvakHW.text
        if naam != "":
            if datum != "":
                if vak != "Selecteer een vak":
                    conn = sqlite3.connect('ScorroDB.db')
                    c = conn.cursor()

                    c.execute("INSERT INTO huiswerk VALUES (:naam, :datum, :beschrijving, :vak)",
                    {
                        'naam': self.root.get_screen('nieuw huiswerk').ids.welkHW.text,
                        'datum': self.root.get_screen('nieuw huiswerk').ids.datumHW.text,
                        'beschrijving': self.root.get_screen('nieuw huiswerk').ids.infoHW.text,
                        'vak': self.root.get_screen('nieuw huiswerk').ids.kiesvakHW.text,
                    })

                    self.root.get_screen('nieuw huiswerk').ids.welkHW.text = ''
                    self.root.get_screen('nieuw huiswerk').ids.datumHW.text = ''
                    self.root.get_screen('nieuw huiswerk').ids.infoHW.text = ''
                    self.root.get_screen('nieuw huiswerk').ids.kiesvakHW.text = 'Selecteer een vak'

                    conn.commit()
                    conn.close()
                else:
                    print("Selecteer een vak")
            else:
                print("Geef je toets een datum")
        else:
            print("Geef je toets een naam")

    def show_huiswerk(self):
        conn = sqlite3.connect('ScorroDB.db')
        c = conn.cursor()

        c.execute("SELECT * FROM huiswerk")
        records = c.fetchall()

        conn.commit()
        conn.close()
        return records

Window.size = (350, 600)

if __name__ == "__main__":
    Scorro().run()


from django.db.models.signals import post_save, pre_delete
from user_accounts.models import UserAccount
from django.dispatch import receiver
from .models import UserProfile, Connection

@receiver(post_save, sender=UserAccount)
def post_save_profile_create(sender, instance, created, **kwargs):
    profile, created = UserProfile.objects.get_or_create(user=instance)
    profile.first_name = instance.first_name
    profile.last_name = instance.last_name
    profile.username = instance.username
    profile.email = instance.email
    profile.save()

@receiver(post_save, sender=Connection)
def post_save_add_connection(sender, instance, created, **kwargs):
     sender_ = instance.sender
     receiver_ = instance.receiver

     if instance.status == "accepted":
        sender_.connections.add(receiver_.user)
        receiver_.connections.add(sender_.user)
        sender_.save()
        receiver_.save()

@receiver(pre_delete, sender=Connection)
def pre_delete_remove_connections(sender, instance, **kwargs):
    sender = instance.sender
    receiver = instance.receiver
    sender.connections.remove(receiver.user)
    receiver.connections.remove(sender.user)
    sender.save()
    receiver.save()


# Programmers 탐욕법(Greedy) 4. 구명보트

# 시간복잡도: O(NlogN)
from collections import deque

def solution(people, limit):
    answer = 0
    queue = deque(sorted(people))
    while queue:
        heavy = queue[-1]
        light = queue[0]
        if len(queue) == 1:
            answer += 1
            break
        if heavy + light <= limit:
            queue.pop()
            queue.popleft()
        else:
            queue.pop()
        answer += 1

    return answer

'''
1. 사람들을 몸무게순으로 정렬한다.
2. 구명보트는 최대 2명만 탈 수 있기 때문에 가장 마지막에 있는 무거운 사람을 먼저 태우고 가능하다면 가장 처음에 있는 가벼운 사람을 같이 태운다.
3. 2번을 반복하는데 가장 마지막과 가장 처음이 같은 사람, 즉 남은 사람이 1명이라면 보트 하나에 태운다.
'''


"""empty message

Revision ID: b22e5138a938
Revises: e6210176f55c
Create Date: 2020-10-29 13:52:55.784625

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = 'b22e5138a938'
down_revision = 'e6210176f55c'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('objectkind',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('name', sa.String(length=64), nullable=True),
    sa.PrimaryKeyConstraint('id')
    )
    op.create_table('validation',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('name', sa.String(length=64), nullable=True),
    sa.Column('status', sa.Integer(), nullable=True),
    sa.PrimaryKeyConstraint('id')
    )
    op.create_table('deploymentvalidation',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('name', sa.String(length=256), nullable=True),
    sa.Column('token_id', sa.Integer(), nullable=True),
    sa.Column('namespace_id', sa.Integer(), nullable=True),
    sa.Column('objectkind_id', sa.Integer(), nullable=True),
    sa.Column('validation_id', sa.Integer(), nullable=True),
    sa.ForeignKeyConstraint(['namespace_id'], ['namespace.id'], ),
    sa.ForeignKeyConstraint(['objectkind_id'], ['objectkind.id'], ),
    sa.ForeignKeyConstraint(['token_id'], ['token.id'], ),
    sa.ForeignKeyConstraint(['validation_id'], ['validation.id'], ),
    sa.PrimaryKeyConstraint('id')
    )
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table('deploymentvalidation')
    op.drop_table('validation')
    op.drop_table('objectkind')
    # ### end Alembic commands ###


__author__       = "Mark Sattolo"
__author_email__ = "epistemik@gmail.com"
__created__ = "2021-01-21"
__updated__ = "2021-01-21"

import csv
import sys
from argparse import ArgumentParser
from datetime import datetime as dt
from cwLibWrappers import chadwick, cwlib
from cwTools import *

class PrintPlayerStats:
    def __init__(self, cwt:MyChadwickTools, logger:logging.Logger):
        self.cwtools = cwt
        self.lgr = logger
        self.lgr.warning(F" Start {self.__class__.__name__}")
        self.event_files = list()
        self.games = {}
        self.note_count = 0

    # void cwbox_print_header(CWGame *game, CWRoster *visitors, CWRoster *home)
    def print_header( self, p_game:pointer, p_vis:pointer, p_home:pointer ):
        self.lgr.info("print_header():\n----------------------------------")

        dn_code = "?"
        day_night = MyCwlib.game_info_lookup(p_game, b"daynight")
        if day_night:
            dn_code = "D" if day_night == "day" else "N" if day_night == "night" else day_night

        game_date = MyCwlib.game_info_lookup(p_game, b"date")
        self.lgr.info(F"game date = {game_date}")
        year, month, day = game_date.split('/')
        game_number = MyCwlib.game_info_lookup(p_game, b"number")
        self.lgr.info(F"game_number = {game_number}")
        game_number_str = "" if game_number == "0" else F", game #{game_number}"

        vis_city = p_vis.contents.city
        vis_city_text = c_char_p_to_str(vis_city)
        self.lgr.info(F"visitor = {vis_city_text}")

        home_city = p_home.contents.city
        home_city_text = c_char_p_to_str(home_city)
        self.lgr.info(F"home = {home_city_text}")

        print(F"\n\t\tGame of {month}/{day}/{year}{game_number_str} -- {vis_city_text} @ {home_city_text} ({dn_code})\n")

    # void cwbox_print_linescore(CWGame *game, CWBoxscore *boxscore, CWRoster *visitors, CWRoster *home)
    def print_linescore( self, p_game:pointer, p_box:pointer, p_vis:pointer, p_home:pointer ):
        self.lgr.info("print_linescore():\n----------------------------------")

        linescore = p_box.contents.linescore
        for t in range(0,2):
            runs = 0
            if t == 0:
                print(F"{c_char_p_to_str(p_vis.contents.city, 16):16}" if p_vis
                      else MyCwlib.game_info_lookup(p_game, b"visteam"), end = '')
            else:
                print(F"{c_char_p_to_str(p_home.contents.city, 16):16}" if p_home
                      else MyCwlib.game_info_lookup(p_game, b"hometeam"), end = '')

            for ix in range(1,32):
                if linescore[ix][t] >= 10:
                    print(F"({linescore[ix][t]})", end = '')
                    runs += linescore[ix][t]
                elif linescore[ix][t] >= 0:
                    print(F"{linescore[ix][t]}", end = '')
                    runs += linescore[ix][t]
                elif t == 0:
                    break
                elif linescore[ix][0] < 0:
                    break
                else:
                    print("x ", end = '')
                    break

                if ix % 3 == 0:
                    print(" ", end = '')

            print(F" -- {runs:2}")

        outs_at_end = p_box.contents.outs_at_end
        if outs_at_end != 3:
            if p_box.contents.walk_off:
                print(F"  {outs_at_end} out{'' if outs_at_end == 1 else 's'} when winning run scored.")
            else:
                print(F"  {outs_at_end} out{'' if outs_at_end == 1 else 's'} when game ended.")

    # void cwbox_print_text(CWGame *game, CWBoxscore *boxscore, CWRoster *visitors, CWRoster *home)
    def print_player_stats( self, p_game:pointer, p_box:pointer, p_vis:pointer, p_home:pointer ):
        self.lgr.info("print_game_summary():\n----------------------------------")

        self.note_count = 0
        slots = [1,1]
        players = list()
        ab = [0,0]
        r  = [0,0]
        h  = [0,0]
        bi = [0,0]
        pa = [0,0]
        bb = [0,0]
        so = [0,0]

        player0 = MyCwlib.box_get_starter(p_box, 0, 1)
        self.lgr.debug(F"type(player0) = {type(player0)}")
        players.insert(0, player0)
        player1 = MyCwlib.box_get_starter(p_box, 1, 1)
        self.lgr.debug(F"type(player1) = {type(player1)}")
        players.insert(1, player1)

        self.print_header(p_game, p_vis, p_home)

        vis_city = c_char_p_to_str(p_vis.contents.city) if p_vis else MyCwlib.game_info_lookup(p_game, b"visteam")
        home_city = c_char_p_to_str(p_home.contents.city) if p_home else MyCwlib.game_info_lookup(p_game, b"hometeam")

        print(F"  {vis_city:18} PA  AB   H  BB  SO  R RBI      {home_city:18} PA  AB   H  BB  SO  R RBI    ")

        while slots[0] <= 9 or slots[1] <= 9 :
            for t in range(0,2):
                if slots[t] <= 9:
                    self.cwtools.print_player(players[t], p_vis if (t == 0) else p_home)
                    # NOTE: misspelling 'battiing' in the python wrapper file
                    batting = players[t].contents.battiing.contents
                    ab[t] += batting.ab
                    r[t]  += batting.r
                    h[t]  += batting.h
                    bb[t] += batting.bb
                    pa[t] += batting.pa
                    so[t] += batting.so
                    if batting.bi != -1:
                        bi[t] += batting.bi
                    else:
                        bi[t] = -1
                    players[t] = players[t].contents.next
                    if not players[t]:
                        # In some National Association games, teams played with 8 players.
                        # This generalization allows for printing boxscores when some batting slots are empty.
                        while slots[t] <= 9 and not players[t]:
                            slots[t] += 1
                            if slots[t] <= 9:
                                players[t] = cwlib.cw_box_get_starter(p_box, t, slots[t])
                else:
                    print(F"{''.ljust(45)}", end = '')
                print("     ", end = ''),
            print("")

        print(F"{''.ljust(20)} --  --  --  --  -- -- -- {''.ljust(24)} --  --  --  --  -- -- --")
        print(F"{''.ljust(20)}{pa[0]:3}{ab[0]:4}{h[0]:4}{bb[0]:4}{so[0]:4}{r[0]:3}", end = '')
        print(F"{bi[0]:3} " if bi[0] >= 0 else "    ", end = '')
        print(F"{''.ljust(24)}{pa[1]:3}{ab[1]:4}{h[1]:4}{bb[1]:4}{so[1]:4}{r[1]:3}", end = '')
        print(F"{bi[1]:3} " if bi[1] >= 0 else "    ")
        print("")

        self.print_linescore(p_game, p_box, p_vis, p_home)
        print("")

        for t in range(0, 2):
            pitcher = MyCwlib.box_get_starting_pitcher(p_box, t)
            if t == 0:
                print(F"  {vis_city:18}   IP  H  R ER BB SO  TP TS GB FB")
            else:
                print(F"  {home_city:18}   IP  H  R ER BB SO  TP TS GB FB")
            while pitcher:
                self.cwtools.print_pitcher( p_game, pitcher, (p_vis if (t == 0) else p_home) )
                pitcher = pitcher.contents.next
            if t == 0:
                print("")

        self.cwtools.print_pitcher_apparatus(p_box)
        print("")

        self.cwtools.print_apparatus(p_game, p_box, p_vis, p_home)
        print("")

# END class PrintPlayerStats

def process_args():
    arg_parser = ArgumentParser(description="Print player combined stats from Retrosheet data for the specified years",
                                prog='main_chadwick_py3.py')
    # required arguments
    required = arg_parser.add_argument_group('REQUIRED')
    required.add_argument('-i', '--player_id', required=True, help="Retrosheet id for a player, e.g. aaroh101, bondb101")
    required.add_argument('-s', '--start', required=True, type=int, help="start year to find stats (yyyy)")
    required.add_argument('-e', '--end', required=True, type=int, help="end year to find stats (yyyy)")
    # optional arguments
    arg_parser.add_argument('-p', '--post', action="store_true", help="find postseason games instead of regular season")
    arg_parser.add_argument('-q', '--quiet', action="store_true", help="NO logging")
    arg_parser.add_argument('-l', '--level', default="INFO", help="set LEVEL of logging output")

    return arg_parser

def process_input_parameters(argx:list):
    args = process_args().parse_args(argx)
    loglevel = "CRITICAL" if args.quiet else args.level
    try:
        getattr( logging, loglevel.strip().upper() )
    except AttributeError as ae:
        print(F"Problem with log level: {repr(ae)}")
        loglevel = "INFO"

    logging.basicConfig(level = loglevel)
    logging.warning(F"process_input_parameters(): Level = {loglevel}\n--------------------------------------")
    logging.info(F"args = \n{args}")

    # TODO: process 'postseason' flag

    playid = args.player_id.strip() if len(args.player_id) >= 8 and \
         args.player_id[:5].isalpha() and args.player_id[5:8].isdecimal() else "maysw001"
    if len(playid) > 8:
        playid = playid[:8]
    logging.warning(F"id = {playid}")

    start = args.start if 1871 <= args.start <= 2020 else 1954
    logging.warning(F"start = {start}")

    end = args.end if 1871 <= args.end <= 2020 else 1955
    if args.end < args.start: end = start
    logging.warning(F"end = {end}")

    return playid, start, end, loglevel

def main_player_stats(args:list):
    lgr = logging.getLogger("PrintPlayerStats")

    playid, start, end, loglevel = process_input_parameters(args)

    lgr.setLevel(loglevel)
    lgr.debug( str(lgr.handlers) )
    lgr.warning(F" id = {playid}; years = {start}->{end}")

    cwtools = MyChadwickTools(lgr)
    player_stats = PrintPlayerStats(cwtools, lgr)
    need_name = True
    fam_name = playid
    giv_name = ""
    # TODO:
    #   get ALL the event files for the requested years
    #   go through each event line for the files and find where player == 'BAT_ID' and save these
    #   go through each file and look for the batting stats of the player and sum
    try:
        for year in range(start, end+1):
            # get the team files
            team_file_name = REGULAR_SEASON_FOLDER + "TEAM" + str(year)
            lgr.info(F"team file name = {team_file_name}")
            with open(team_file_name, newline = '') as team_csvfile:
                team_reader = csv.reader(team_csvfile)
                for trow in team_reader:
                    rteam = trow[0]
                    lgr.info(F"Found team {rteam}")
                    # get the rosters
                    if need_name:
                        roster_file = ROSTERS_FOLDER + rteam + str(year) + ".ROS"
                        lgr.debug(F"roster file name = {roster_file}")
                        if not os.path.exists(roster_file):
                            raise FileNotFoundError(F"CANNOT find roster file {roster_file}!")
                        with open(roster_file, newline = '') as roster_csvfile:
                            ros_reader = csv.reader(roster_csvfile)
                            for rrow in ros_reader:
                                if playid == rrow[0]:
                                    fam_name = rrow[1]
                                    giv_name = rrow[2]
                                    need_name = False
                                    break
                    # find and store the event file paths
                    event_file = REGULAR_SEASON_FOLDER + str(year) + rteam + ".EV" + trow[1]
                    if not os.path.exists(event_file):
                        raise FileNotFoundError(F"CANNOT find event file {event_file}!")
                    player_stats.event_files.append(event_file)

        lgr.info(F"name = {giv_name} {fam_name}")
        lgr.info(F"found {len(player_stats.event_files)} event files")
        for item in player_stats.event_files:
            lgr.debug(item)

        event_rows = 0
        # get all the games for the requested team in the supplied date range
        for evteam in player_stats.event_files:
            lgr.info(F"found events for team = {evteam}")
            cwgames = chadwick.games(evteam)
            for game in cwgames:
                game_id = game.contents.game_id.decode(encoding='UTF-8')
                game_date = game_id[3:11]
                lgr.debug(F" Found game id = {game_id}; date = {game_date}")

                events = chadwick.process_game(game)
                results = tuple(events)
                for row in results:
                    if row['BAT_ID'] == playid:
                        event_rows += 1
                        # player_stats.print_player_stats()
        lgr.info(F"found {event_rows} event rows for {giv_name} {fam_name} in {start} through {end}.")

    except Exception as ex:
        lgr.exception(F"Exception: {repr(ex)}")

if __name__ == "__main__":
    run_start_time = dt.now()
    if '-q' not in sys.argv:
        logging.critical(F"Run Start time = {run_start_time.strftime('%Y-%m-%d %H:%M:%S')}")
    main_player_stats(sys.argv[1:])
    if '-q' not in sys.argv:
        run_time = (dt.now() - run_start_time).total_seconds()
        logging.critical(F" Running time = {(run_time // 60)} minutes, {(run_time % 60):2.3} seconds")
    exit()


import os

import matplotlib.pyplot as plt

os.chdir('C:\\Users\\oscar\\OneDrive\\Bureau\\IF29\\Tweet Worldcup\\raw')
os.getcwd()

import json
import pandas as pd
import numpy as np
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
from sklearn.cluster import KMeans

os.chdir('C:\\Users\\oscar\\OneDrive\\Bureau\\IF29')
df = pd.read_csv('finalDF.csv')

print(df.shape)

#On centre réduit les données
s_sc = StandardScaler()
df_processed = s_sc.fit_transform(df)

#On réalise l'ACP
modelPCA = PCA(n_components=2)
df_reduced = modelPCA.fit_transform(df_processed)

plt.scatter(df_reduced[:,0],df_reduced[:,1])
plt.show()

#Elbow method
inertia = []
K_range = range(1, 6)
for i in K_range:
    modelElbow = KMeans(n_clusters=i).fit(df_reduced)
    inertia.append(modelElbow.inertia_)

plt.plot(K_range, inertia)
plt.xlabel('nb de clusters')
plt.ylabel('Inertie')
plt.show()

#KMeans
modelKMeans = KMeans(n_clusters=3)
df_KMeans = modelKMeans.fit(df_reduced)
label = modelKMeans.fit_predict(df_reduced)

filtered_label0 = df_reduced[label == 0]
filtered_label1 = df_reduced[label == 1]
filtered_label2 = df_reduced[label == 2]

plt.scatter(filtered_label0[:,0] , filtered_label0[:,1] , color = 'red')
plt.scatter(filtered_label1[:,0] , filtered_label1[:,1] , color = 'black')
plt.scatter(filtered_label2[:,0] , filtered_label2[:,1] , color = 'green')
plt.show()
'''
plt.scatter(df_reduced[:,0], df_reduced[:,1], c=modelKMeans.predict(df_reduced), label=modelKMeans.labels_)
plt.scatter(modelKMeans.cluster_centers_[:,0], modelKMeans.cluster_centers_[:,1], c='r')
plt.show()
'''
'''
# Opening JSON file
tweets = []
for i in range(20):
  for line in open('raw'+ str(i) +'.json', 'r', encoding='utf8'):
      tweets.append(json.loads(line))

df = pd.DataFrame(tweets)
print(df.head())
print(df.describe())
#Creation de l'indicateur de followers et abonnements et du ratio
userdf = df['user']
indicateurDfFriendsCount = np.zeros(len(userdf))
indicateurDfFollowersCount = np.zeros(len(userdf))
indicateurDfRatio = np.zeros(len(userdf))

for i in range(len(userdf)):
    for key, value in userdf[i].items():
        if (key == "friends_count"):
            #print(key, value)
            indicateurDfFriendsCount[i] = value
        elif (key == "followers_count"):
            indicateurDfFollowersCount[i] = value

for i in range(len(indicateurDfFollowersCount)):
    if (indicateurDfFriendsCount[i] != 0):
        indicateurDfRatio[i] = indicateurDfFollowersCount[i] / indicateurDfFriendsCount[i]
    else:
        indicateurDfRatio[i] = indicateurDfFollowersCount[i]

tweetsText = np.array([])

for i in range(len(df)):
   tweetsText = np.append(tweetsText, "")

dfText = df['text']

for i in range(len(df)):
    tweetsText[i] = dfText[i]

indicateurDfLongueurTweets = np.zeros(len(df))

for i in range(len(df)):
    indicateurDfLongueurTweets[i] = len(tweetsText[i])

#Création de l'indicateur nombre de hahtag
entitiesdf = df['entities']
indicateurNbHashtags = np.zeros(len(userdf))

for i in range(len(entitiesdf)):
    indicateurNbHashtags[i] = len(entitiesdf[i]['hashtags'])

#Création de l'indicateur nombre de URLs
indicateurNbURLs = np.zeros(len(userdf))
for i in range(len(entitiesdf)):
    indicateurNbURLs[i] = len(entitiesdf[i]['urls'])

#Aggressiveness
# "users" stockera les données nécessaire au calcul de l'agressivité pour chaque profile
users = {}
# On execute la requête et pour chacun des tweets, on conserve les données qui nous intéressent
for i in range(len(df)):
    # Si l'utilisateur n'a pas encore été rencontré, on l'ajoute à notre dictionnaire users
    if df.iloc[i, :]['user'].get('id') not in users:
        users[df.iloc[i, :]['user'].get('id')] = {}
        users[df.iloc[i, :]['user'].get('id')]['timestamp'] = []
        users[df.iloc[i, :]['user'].get('id')]['outgoinglinks'] = []
        users[df.iloc[i, :]['user'].get('id')]['outgoinglinks'].append(len(df.iloc[i, :]['entities'].get('urls')))
        users[df.iloc[i, :]['user'].get('id')]['timestamp'].append(int(df.iloc[i, :]['timestamp_ms']))

    # Sinon on ajoute l'id d'utilisateur au dictionnaire users
    else:
        users[df.iloc[i, :]['user'].get('id')]['timestamp'].append(int(df.iloc[i, :]['timestamp_ms']))
        users[df.iloc[i, :]['user'].get('id')]['outgoinglinks'].append(len(df.iloc[i, :]['entities'].get('urls')))

print("Parcours des utilisateurs fini")

# On détermine l'agressivité de chacun des profiles
for user in users:
    users[user].get('timestamp').sort(reverse=True)
    timestamp = users[user].get('timestamp')
    users[user].get('outgoinglinks').sort(reverse=True)
    outgoinglinks = users[user].get('outgoinglinks')
    users[user]['frequenceTweet'] = 1
    users[user]['frequenceFriends'] = 0
    nbOutgoingLinks = 0
    # S'il y a plus d'un tweet de retenu, on additionne les écarts entre chaque tweet
    if len(timestamp) > 1:
        diffTime = timestamp[0] - timestamp[len(timestamp) - 1]
        # On convertit le temps en h
        diffTime = diffTime / float(3600000) + 0.000000000000001
        users[user]['frequenceTweet'] = len(timestamp) / diffTime
        for i in range(len(outgoinglinks)):
            nbOutgoingLinks += outgoinglinks[i]
        users[user]['frequenceFriends'] = nbOutgoingLinks / diffTime

    users[user]['agressivité'] = (users[user]['frequenceFriends'] + users[user]['frequenceTweet']) / 350

indicateurAgressivite = np.array([])

for user in users:
    indicateurAgressivite = np.append(indicateurAgressivite, users[user]['agressivité'])

print(indicateurAgressivite)

#VISIBILITY
moyLengthHashtags = 11.6
moyLengthMention = 11.4

# "usersVis" stockera les données nécessaire au calcul de l'agressivité pour chaque profile
usersVis = {}
# On execute la requête et pour chacun des tweets, on conserve les données qui nous intéressent
for i in range(len(df)):
    # Si l'utilisateur n'a pas encore été rencontré, on l'ajoute à notre dictionnaire users
    if df.iloc[i, :]['user'].get('id') not in usersVis:
        usersVis[df.iloc[i, :]['user'].get('id')] = {}
        usersVis[df.iloc[i, :]['user'].get('id')]['hashtags'] = []
        usersVis[df.iloc[i, :]['user'].get('id')]['user_mentions'] = []
        usersVis[df.iloc[i, :]['user'].get('id')]['user_mentions'].append(len(df.iloc[i, :]['entities'].get('user_mentions')))
        usersVis[df.iloc[i, :]['user'].get('id')]['hashtags'].append(len(df.iloc[i, :]['entities'].get('hashtags')))

    # Sinon on ajoute l'id d'utilisateur au dictionnaire users
    else:
        usersVis[df.iloc[i, :]['user'].get('id')]['hashtags'].append(len(df.iloc[i, :]['entities'].get('hashtags')))
        usersVis[df.iloc[i, :]['user'].get('id')]['user_mentions'].append(len(df.iloc[i, :]['entities'].get('user_mentions')))

print("Parcours des utilisateurs fini pour visibilité")

for user in usersVis:
    nbTotalHashtags = usersVis[user].get('hashtags')
    nbTotalUser_mentions = usersVis[user].get('user_mentions')
    avgHashtags = sum(nbTotalHashtags) / len(nbTotalHashtags)
    avgUser_mentions = sum(nbTotalUser_mentions) / len(nbTotalUser_mentions)
    visibilité = ((avgHashtags * moyLengthHashtags) + (avgUser_mentions * moyLengthMention)) / 140
    usersVis[user]['visibilité'] = visibilité
'''


import sys
sys.path.append("/media/alesya/381A50EF72E26C1A/Yandex-disk/KLayout-python-shamil_dev_sfs/")

import pya
from math import sqrt, cos, sin, atan2, pi, copysign
from pya import Point, DPoint, DSimplePolygon, SimplePolygon, DPolygon, Polygon, Region, Instance, CellInstArray
from pya import Trans, DTrans, CplxTrans, DCplxTrans, ICplxTrans

from ClassLib.BaseClasses import *

class AlignCross(ElementBase):

    def __init__(self, origin, ebeam, trans_in=None):

        self._ebeam = ebeam
        self._origin = origin

        super().__init__(origin, trans_in)

        self.start = self.connections[0]
        self.end = self.connections[1]
        self.alpha_start = self.angle_connections[0]
        self.alpha_end = self.angle_connections[1]

    def init_regions(self):

        if self._ebeam:
            w1 = 4e3/2*4
            w2 = 2e3/2*4
            w3 = 1e3/2*4
            w4 = 0.5e3/2*2
        else:
            w1 = 8e3/2*4
            w2 = 6e3/2*4
            w3 = 4e3/2*4
            w4 = 2e3/2*2

        l = 10e3*4

        empty_points = [DPoint(-50e3*4,-50e3*4),
                  DPoint(-50e3*4,50e3*4),
                  DPoint(50e3*4,50e3*4),
                  DPoint(50e3*4,-50e3*4)]
        empty_poly = DSimplePolygon(empty_points)
        empty_region = Region(empty_poly)

        metal_points = [DPoint(-l*4,-w1),
                  DPoint(-l*4,w1),
                  DPoint(-l*3,w1),
                  DPoint(-l*3,w2),
                  DPoint(-l*2,w2),
                  DPoint(-l*2,w3),
                  DPoint(-l,w3),
                  DPoint(-l,w4),

                  DPoint(-w4,w4),
                  DPoint(-w4,l),
                  DPoint(-w3,l),
                  DPoint(-w3,2*l),
                  DPoint(-w2,2*l),
                  DPoint(-w2,3*l),
                  DPoint(-w1,3*l),
                  DPoint(-w1,4*l),
                  DPoint(w1,4*l),
                  DPoint(w1,3*l),
                  DPoint(w2,3*l),
                  DPoint(w2,2*l),
                  DPoint(w3,2*l),
                  DPoint(w3,l),
                  DPoint(w4,l),
                  DPoint(w4,w4),

                  DPoint(l,w4),
                  DPoint(l,w3),
                  DPoint(l*2,w3),
                  DPoint(l*2,w2),
                  DPoint(l*3,w2),
                  DPoint(l*3,w1),
                  DPoint(l*4,w1),
                  DPoint(l*4,-w1),
                  DPoint(l*3,-w1),
                  DPoint(l*3,-w2),
                  DPoint(l*2,-w2),
                  DPoint(l*2,-w3),
                  DPoint(l,-w3),
                  DPoint(l,-w4),
                  DPoint(w4,-w4),

                  DPoint(w4,-l),
                  DPoint(w3,-l),
                  DPoint(w3,-2*l),
                  DPoint(w2,-2*l),
                  DPoint(w2,-3*l),
                  DPoint(w1,-3*l),
                  DPoint(w1,-4*l),
                  DPoint(-w1,-4*l),
                  DPoint(-w1,-3*l),
                  DPoint(-w2,-3*l),
                  DPoint(-w2,-2*l),
                  DPoint(-w3,-2*l),
                  DPoint(-w3,-l),
                  DPoint(-w4,-l),
                  DPoint(-w4,-w4),

                  DPoint(-l,-w4),
                  DPoint(-l,-w3),
                  DPoint(-l*2,-w3),
                  DPoint(-l*2,-w2),
                  DPoint(-l*3,-w2),
                  DPoint(-l*3,-w1)]

        metal_poly = DSimplePolygon(metal_points)
        self.metal_region.insert(metal_poly)

        self.empty_region.insert(empty_region - self.metal_region)

        self.connections = [DPoint(0,0), DPoint(0, 0)]
        self.angle_connections = [pi,0]

class AlignStructure(ElementBase):

    def __init__(self, origin, ebeam, trans_in=None):

        self._ebeam = ebeam
        self._origin = origin

        super().__init__(origin, trans_in)

        self.start = self.connections[0]
        self.end = self.connections[1]
        self.alpha_start = self.angle_connections[0]
        self.alpha_end = self.angle_connections[1]

    def init_regions(self):

        if self._ebeam:
            dw = 2e3*4
        else:
            dw = 3e3*4

        w = 2e3*4

        l = (w*7+dw*6)/2

        lengths = [3e3*4, 6e3*4, 8e3*4, 9e3*4, 8e3*4, 6e3*4, 3e3*4]

        empty_points = [DPoint(0,-l),
                  DPoint(0,l),
                  DPoint(9e3*4,l),
                  DPoint(9e3*4,-l)]
        empty_poly = DSimplePolygon(empty_points)
        empty_region = Region(empty_poly)

        l0 = -l

        metal_region = Region(DSimplePolygon([]))

        for l1 in lengths:
            metal1 = Region(DSimplePolygon([DPoint(0,l0),
                    DPoint(0,l0+w),
                    DPoint(l1, l0+w),
                    DPoint(l1,l0)]))

            metal_region += metal1
            l0 = l0+w+dw

        self.metal_region.insert(metal_region)

        self.empty_region.insert(empty_region - metal_region)

        self.connections = [DPoint(0,0), DPoint(0, 0)]
        self.angle_connections = [pi,0]

class AlignmentMark(ElementBase):

    def __init__(self, origin, trans_in=None):

        self._origin = origin

        super().__init__(origin, trans_in)

        self.start = self.connections[0]
        self.end = self.connections[1]
        self.alpha_start = self.angle_connections[0]
        self.alpha_end = self.angle_connections[1]

    def init_regions(self):

        self.metal_regions["photo"] = Region()
        self.empty_regions["photo"] = Region()
        self.metal_regions["ebeam"] = Region()
        self.empty_regions["ebeam"] = Region()

        ebeamCross = AlignCross(DPoint(0,0), True)
        photoCross = AlignCross(DPoint(0,0), False)

        ebeamStr = AlignStructure(DPoint(25e3*4,25e3*4), True, trans_in=DTrans.R180)
        photoStr = AlignStructure(DPoint(25e3*4,25e3*4), False)

        ebeamStr1 = AlignStructure(DPoint(25e3*4,-25e3*4), True, trans_in=DTrans.R270)
        photoStr1 = AlignStructure(DPoint(25e3*4,-25e3*4), False, trans_in=DTrans.R90)

        self.metal_regions["ebeam"].insert(ebeamCross.metal_region+ebeamStr.metal_region+ebeamStr1.metal_region)
        self.empty_regions["ebeam"].insert(ebeamCross.empty_region-ebeamStr.metal_region-ebeamStr1.metal_region)
        self.metal_regions["photo"].insert(photoCross.metal_region+photoStr.metal_region+photoStr1.metal_region)
        self.empty_regions["photo"].insert(photoCross.empty_region-photoStr.metal_region-photoStr1.metal_region)

        self.connections = [DPoint(0,0),
                            DPoint(0,0)]
        self.angle_connections = [pi,0]


import numpy as np

import nibabel as nib
from scipy import ndimage

def read_nifti_file(filepath):
    """Read and load volume"""
    # Read file
    scan = nib.load(filepath)
    # Get raw data
    scan = scan.get_fdata()
    return scan

def normalize(volume):
    """Normalize the volume"""
    min = np.min(volume)
    max = np.max(volume)
    volume = (volume) / (max)
    volume = volume.astype("float32")
    return volume

def pretty_dict(dict_x):
    """

    :param dict_x:
    :return:
    """
    str_line = ''.join(['-'] * 100)
    str_dict = f"{str_line}"
    for key, val in dict_x.items():
        str_dict += f"\n{key}: {val} ({type(val)})"
    str_dict += f"\n{str_line}"
    return str_dict

def resize_volume(img,
                  desired_width=128,
                  desired_height=128,
                  desired_depth=32,
                  ):
    """Resize across z-axis"""
    # Set the desired depth

    # Get current depth
    current_depth = img.shape[-1]
    current_width = img.shape[0]
    current_height = img.shape[1]
    # Compute depth factor
    depth = current_depth / desired_depth
    width = current_width / desired_width
    height = current_height / desired_height
    depth_factor = 1 / depth
    width_factor = 1 / width
    height_factor = 1 / height
    # Rotate
    img = ndimage.rotate(img, 90, reshape=False)
    # Resize across z-axis
    img = ndimage.zoom(img, (width_factor, height_factor, depth_factor), order=1)
    return img

def process_scan(path,
                 desired_width=128,
                 desired_height=128,
                 desired_depth=32,
                 ):
    """Read and resize volume"""
    # Read scan
    volume = read_nifti_file(path)
    # Normalize
    volume = normalize(volume)
    # Resize width, height and depth
    volume = resize_volume(volume,
                           desired_width,
                           desired_height,
                           desired_depth)
    return volume


import bpy
import os
import numpy
#from .config import IMAGES_DIR

#References:
#   https://docs.blender.org/api/blender_python_api_2_71_release/bpy.types.Image.html
#   https://blender.stackexchange.com/questions/13422/crop-image-with-python-script/13516#13516
#   https://blender.stackexchange.com/questions/3673/why-is-accessing-image-data-so-slow/3678#3678
#   https://blender.stackexchange.com/questions/15890/is-it-possible-to-edit-images-programmatically-with-the-blender-api

class loadImage:
    def __init__(self):
        self.loadedImages = {}

    def __del__(self):
        # destructor that removes images added by the addon, this will keep the blend file tidy despite running the addon multiple times.
        def cleanupImages():
            for filename in self.loadedImages:
                # print("filename: "+filename)
                for image in bpy.data.images:
                    # print("image: "+image.name)
                    if os.path.basename(filename)==image.name:
                        bpy.data.images.remove(image)
        # use the call below to disable this function if you're testing image updating
        cleanupImages()
        print("Cleaned up images")

    # Optimizations thanks to https://blender.stackexchange.com/questions/3673/why-is-accessing-image-data-so-slow/3678#3678
    # Blender stores images as 1D arrays, ordered as described in https://blender.stackexchange.com/questions/13422/crop-image-with-python-script/13516#13516
    # This function generates a 2D array of (r,g,b,a) tuples from said 1D list
    # Calling the result at index [i][j] will return the pixel in the i-th row from the bottom, j-th column from the left
    # i.e. where i=j=0 is the bottom-left pixel.
    def blendImage2xyImage(self, blenderImage):
        print("Loading image:"+blenderImage.name)
        x = blenderImage.size[0]
        y = blenderImage.size[1]
        image = numpy.empty(shape=(x,y), dtype=(float,4))
        pixels = blenderImage.pixels[:]

        index = 0
        # iterate over every row
        for i in range(y):
            # iterate over every pixel in a row
            for j in range(x):
                r = pixels[index]
                g = pixels[index+1]
                b = pixels[index+2]
                a = pixels[index+3]
                # Apparently, some images have "pre-multiplied" alphas...
                if a > 0 and blenderImage.alpha_mode == "PREMUL":
                    r = r/a
                    g = g/a
                    b = b/a
                image[i][j] = (r,g,b,a)
                index += 4
        return image

    # update a given blender image with a given xyImage
    def xyImage2blendImage(self, xyImageName, xyImageContents, blendImage):
        print("Saving xyImage: "+xyImageName)
        xyPixels = []
        for row in xyImageContents:
            for pixel in row:
                (r,g,b,a) = pixel
                if a > 0 and blendImage.alpha_mode == "PREMUL":
                    r *= a
                    g *= a
                    b *= a
                xyPixels.append(r)
                xyPixels.append(g)
                xyPixels.append(b)
                xyPixels.append(a)
        blendImage.pixels[:] = xyPixels
        blendImage.update()

    def createArrays(self):
        return

    # Save the xy images to the right blend images
    def saveImages(self):
        for xyImage in self.loadedImages:
            for blendImage in bpy.data.images:
                # print("blend image image: "+blendImage.name)
                if os.path.basename(xyImage)==blendImage.name:
                    self.xyImage2blendImage(xyImage, self.loadedImages[xyImage], blendImage)

    def setupFilepath(self, imageFolder):
        # This will get the currently opened blender file's directory
        # Idea thanks to https://docs.blender.org/api/current/bpy.data.html
        # BlendFileDir = bpy.data.filepath
        # print(BlendFileDir)
        # print(BlendFileDir.rfind('\\'))
        blendFileDir = bpy.data.filepath[:bpy.data.filepath.rfind('\\')+1]
        # print(blendFileDir)
        blendFileDir += imageFolder + "\\"
        # print(blendFileDir)
        return blendFileDir

    def openImage(self, imageFile):
        image = bpy.data.images.load(imageFile)
        imageResult = self.blendImage2xyImage(image)
        self.loadedImages[imageFile] = imageResult
        return imageResult

    # Open all the images in a specified folder
    def openAllImagesInFolder(self, folder):
        # This loops over every file in the identified images directory
        imageDir = self.setupFilepath(folder)
        for filename in os.listdir(imageDir):
            # print(self.imageDir+filename)
            imageFile = imageDir+filename
            image = bpy.data.images.load(imageFile)
            self.loadedImages[imageFile] = self.blendImage2xyImage(image)
            # print("Loaded")
            # print(image)
        # print(self.loadedImages[0])'''

if __name__ == "__main__":
    # Test function
    # calculate the runtime of this script
    import time
    startTime = time.time()
    print("Running script...")

    # function scope so that the image class gets deleted automatically (so the destructor will be called)
    def main():
        # Test the script itself
        image = loadImage()
        image.openAllImagesInFolder("CubeHard")
        # del image

    main()
    stopTime = time.time()
    print("Script took:",stopTime-startTime)