#AUTOR : Arnaud HINCELIN
#Date : 07/2020

print("==========NUMPY AND MACHINE LEARNING===========")
"""
On commence !
package le plus important, numpy avec l'objet le plus puissant :
tableau à n dimensiosn = ND Array
C'est dans ce tableau que l'on va entrer nos données pour entrainer notre IA
"""

#OBJET ND ARRAY
"""
Créer tableaux à 1(ligne) , 2(matrice) ou 3 dimensions (cube)

Tableau 1D ARRAY
ressemble à une liste (aussi des index)
mais ndarray bcp plus puissant : rapide, moins mémoire, plus de méthodes

Tableau 2D ARRAY
très utilisé, comparable à tableau excel
On peut stocker pixels d'une image noir et blanc, valeur de 0 à 255

Tableau 3D ARRAY
Alors on peut traiter images en couleur car superposition de 3 tableaux 2D
les 3 tableux 2D : vert, bleu ,rouge

Chaque tableau aura des méthodes et attributs :
attribut les plus utiliés :

#Attribut ndim
renvoi la dimension

#Attribut shape
renseigne sur la forme, combien lignes et colones.
shape renvoi un tuple:
séquence=on peut accéder aux diff valeurs (shape[n])
tuple=non modifiable (immutable)
tab à 1 dimension de 2 lignes
alors : shape=(2)
shape[0]=2
tab à 2 dimension de 3 colones et 2 lignes
alors : shape=(2,3)
shape[1]=3
tab à 3 dimension de 3 colones et 2 lignes et 2 tableaux superposés
alors : shape=(2, 3, 2)
shape[2]=2

"""

print("=====CONSTRUCTOR AND TABLEAUX======")

#Constructeurs de tableaux
"""
np.array(objet)
np.zeros(shape) #créer tableau init à 0
np.full(shape, value) #créer tableau init à value
np.ones(shape) #créer tableau init à 1
np.eye(n) #créé mat diagonale, avec n diagonales

np.random.randn(shape) #créer tableau de valeurs aléatoires
ces nbs aléa sont données selon la loi normale centrée en 0;proba plus importante pr nbs proches de 0

np.linspace(début,fin,quantité) #créé tab 1D dont on précise val de début
et val de fin, et la quantité déléments t.riés par ordre

np.arange(début, fin, pas) #créé tab 1D dont on précise val de début et
val de fin, et le pas. On connait pas la qté d'éléments

Attention; shape est un tuple ! s'écrit np.zeros((2,3))
Sauf random : np.random.randn(2,3)

"""
import numpy as np
A=np.array([1,2,3])

A.shape #renvoi la forme
A.ndim #renvoi la dimension
A.size #renvoi nb éléments

B=np.zeros((3,2,3))#tableau de dim 3 init à 0
C=np.random.randn(2,3) #tableau de dim 2 init aléatoire
D=np.eye(3)
E=np.linspace(0,10,20,dtype=np.float16)

#Types de données
"""
On peut choisir le type de données des tableaux
np.float, np.int,... avec la param (dtype=np.float)
Mais aussi la place mémoire de chaque varibale
np.int8 ; np.int16 ; ... ; np.uint8 ; ...
mais,
np.int8 = moins précis, plus rapide
np.int64 = plus précis, mois rapide

"""
E=np.linspace(0,10,20,dtype=np.float16)

#Methodes importantes
A=np.zeros((3,2))
B=np.ones((3,2))

"""
#Coller 2 tableaux
np.hstack((Array A, Array B))#coller 2 tab horizontale, A et B même nb lignes
np.vstack((Array A, Array B))#coller 2 tab verticale, A et B même nb colones
np.concatenate((Array A, Array B), axis)#axis(axe de collage) = 0(axe verti) ou 1(axe hori)

#Remanipuler forme d'un tableau
Il faut même nb éléments !
Array.np.reshape((a,b))
Pb des listes dim 1D : shape (3,)=> rien après virgule
pr éviter pb en algo, mettre un 1 pour remplacer espace après virgule
Faire reshape : Array.reshape((Array.shape[0], 1))

#Applatir un tableau en un tableau en 1D
Array.ravel()
[[0. 0.]
 [0. 1.]	==>> [0. 0. 0. 1. 1. 1.]
 [1. 1.]]

Si on veut ensuite utiliser (3,) => Array.squeeze()

"""
C=np.hstack((A,B)) #A et B même shape[0] !
C=np.vstack((A,B)) #A et B même shape[1] !
C=np.concatenate((A,B), axis=0) #comme hstack
C=C.reshape((3,4)) #passe d'un (6,2) à (3,4)

H=np.array([1,2,3])
H.shape #affiche (3,)
H=H.reshape((H.shape[0],1))
H.shape #affiche (3,1)
H=H.squeeze()
H.shape #affiche (3,)

I=A.ravel() #affiche [0. 0. 0. 0. 0. 0.]

print("==EXO==")
def initalisation(m,n):
	#m nb lignes
	#n nb colones
	#retourne matrice aléa (m, n+1)
	#colone tout à droite remplie de 1
	#indice : reshape, concatenate, randam.randn()

	#on créer matrice mxn aléatoire
	X = np.random.randn(m, n)
	#on créer colone de 1 avec même nb ligne que X
	#shape = (X.shape[0],1) car unique colonne
	Col = np.ones((X.shape[0],1))
	#on la concatene verticale avec colone
	X = np.concatenate((X, Col), axis=1)

	return X
#print(initalisation(2,2))

print("=====NUMPY INDEXING SLICING SUBSETTING======")
"""
Idée:naviguer dans les tableaux avec les techniques vues.
Comme liste, mais ici galère car espace à n dimensions
En DataSciences, souvent tableaux 2D
Axe 0 = lignes et Axe 1 = colonnes
Pour pas perdre, naviguer seulement sur un axe à la fois; une seule donnée change

"""

#INDEXING
"""
Donc se déplacer avec INDEXING;

A[ligne, colonne]
depl vers la droite A[0,1]
depl vers le bas A[1,0]

"""
A=np.array([[1,2,3],[4,5,6],[7,8,9]])
print(A[0,0]) #affiche 1

#SLICING
"""
#bases
A[début:fin(lignes), début:fin(colonnes)]

toute la colone n = A[:,n]
toute la ligne n = A[n,:]
mais ligne n s'affiche aussi A[n]

séléctionner un bloc :
A[0:n, 0:n] de 0 à n-1 lignes et de 0 à n-1 colonnes
A[1:3, 1:] affiche [[_ _ _] lignes 1 et 2
				    [_ * *] colonnes 1 à la fin
				    [_ * *]]

mais si on connait pas fin du tableu :
dernier:-1 av-dernier:-2
A[-1] : dernière ligne
A[:, -2:] = ttes lignes , colonnes à partie de avant-dernière

Séctionner un bloc pour lui donner une valeur:
A[0:2, 0:2] = 10
A maintenant égal à [[10,10,3][10,10,6][7,8,9]]

#intégrer le pas
A[début:fin:pas(lignes), début:fin:pas(colonnes)]
Ex : séléctionner ces données :
								[[* _ * _ *]
							     [_ _ _ _ _]
							     [* _ * _ *]
							     [_ _ _ _ _]
							     [* _ * _ *]]
A[::2, ::2]

"""
A[:,0] #affiche [1,4,7]
A[0,:] #affiche [1,2,3]
A[0] #affiche [1,2,3]
A[0:2, 0:2] #affiche [[1,2][4,5]]
A[-2] #affiche avant dernière ligne
A[1:3, 1:] #affiche [[5,6][8,9]]

B=np.zeros((4,4)) #B=mat 4x4 de 0
B[1:3, 1:3] = 1 #B=mat 4x4 de 0 mais carré de 1 au centre

C=np.zeros((5,5)) #C=mat 5x5 de 0
C[::2,::2]=1 #1 placé sur ttes lignes et colonnes avec pas de 2

#BOOLEAN INDEXING
"""
On peut obtenir un tableau de booléen avec une condition :
A tableau 5x5 d'entiers de 0 à 9
A<5 renvoi un tableau de booléen true ou false
Cela s'appelle un MASQUE
Utile pour images, filter données d'un tableau de meme dim

#Attention, A[A<5] = tab 1D des valeurs
#			A(A<5) = tab 2D des booléens
"""

A=np.random.randint(0, 10, [5,5])#tab de 5x5 avec aléa de 0 à 9
A<5 #tableau de booleen sur cette condition
A[(A<5) & (A>2)]=10 #remplacer valeurs par 10 si conditions vérifiées

#Ex du masque avec image:
Pixels=np.random.randint(0, 255, [1024,720])
Pixels[Pixels>200]=255 #pixels avec luminosité élevée sont égaux à 255
#Ex du filtrage donnant une liste (tab 1D)
A[A<5] #tableau 1D des données de A accordant la condition

#Attention
B=np.random.randn(5,5)
print(B[A<5])#tableau 1D
#données viennent de B mais séléctionnés par Booléen indexing de A

print("==EXO==")
"""
zoomer de 0,75 sur le centre de l'image puis appliquer un filtre dessus
en augmentant luminosité ou baissant (>200 ou <200)
photo chargeable directzement depuis module scipy
"""

#Charger le photo
"""
face est la photo = tableau numpy (verif avec type(face))
face = misc.face()
plt.imshow(face)
plt.show() #afficher image

#si charger en noir et blanc:
face = misc.face(gray=True)
plt.imshow(face, cmap=plt.cm.gray)
plt.show() #afficher image

face.shape #affiche (768,1024,3) => tab à 3D car 3 couleurs

pr exo, charge en noir et blanc, dc en 1D

from scipy import misc
import matplotlib.pyplot as plt
face = misc.face(gray=True)
plt.imshow(face, cmap=plt.cm.gray)
plt.show() #afficher image
"""
#zoomer
"""
utiliser dimensions avec le reshape et enregistrer
dimensions ds des var et les utiliser dans techinique
de slicing
"""
#print(face.shape)affiche (768,1024)
#zoom de 0,75==>> (576,768)

#CODE#
#importer imageet la mettre en noir et blanc
from scipy import misc
import matplotlib.pyplot as plt
face = misc.face(gray=True)

#Zoomer
print(face.shape)#affiche (768,1024)
#zoom de 0,75==>> (576,768)
#(768-576)/2 = 96 et (1024-768)/2 = 128
face=face[96:672, 128:896]

#Masquer avec booléen indexing
face[face>170] = 255
face[face<170] = 0

#print(face)

#Afficher image zoomée et masquée
plt.imshow(face, cmap=plt.cm.gray)
#plt.show() #afficher image

print("==========NUMPY STATS AND MATHS=============")
"""
faire des stats et maths
import numpy as np
"""
print("=====MATHEMATIQUES======")

#METHODES NDARRAY
"""
sum() = faire somme des éléments
cumsum() = renvoi tab de somme cummulée
prod() = produit des éléments du tableau
cumprod() = produit cumulé
sort() = trier tableau
argsort() = renvoi tab des index du tableau trié
max() = renvoi max
argmax() = renvoi index de position du max
min() = renvoi min
argmin() = renvoi index de position du min
mean() = renvoi moyenne
... bcp autres

Appliquer methode sur les axe: 0(vertical) et 1(hori)
ex avec sum
sum(axis=0):tab(dim=colonnes)avec somme de chaque colonne
sum(axis=1):tab(dim=lignes) avec somme de chaque ligne

"""
np.random.seed(0) #fixer le random
A = np.random.randint(0,10,[2,3])#A=[[5,0,3], [3,7,9]]
A.sum() #affiche 27
A.sum(axis=1) #affiche colonne [8 19]
A.sum(axis=0) #affiche ligne [8 7 12]
A.argmin(axis=0) #affiche [1 0 0]
A.argsort()

#METHODES MATHS NORMALES
"""
sin()
exp()
...bcp autres

s'applique comme ça : np.exp(A) = renvoi tab avec valeurs expo

"""
np.exp(A)

print("=====STATISTIQUES======")

#METHODES NDARRAY
"""
mean() = moyenne du tab
var() = variance du tab
std() = ecart type du tableau
...bcp autres

"""
A.mean() #affiche 4.5

#METHODES STATS NORMALES
"""
Cours stats:
Coefficicent de corrélation linéaire R :
- nb qui permet de calculer l'intensité d'un lien linéaire entre deux variables
- R sans unité, situé entre -1 et 1
- R>0;lien linéaire positif (si X aug, Y aug) et R<0 ; lien linéaire négatif (si X aug, Y dim)
- plus R proche de 1 ou -1, plus lien fort
Droite régression:
- droite minimisant somme des carrés des distances verticlaes entre les points et elle même

Matrice de corrélation:
- mat qui indique les liens de coorélation entre lignes ou colonnes du tableau
- mat est symétrique et sa diago est constitué de 1

np.corrcoef(A) #trace mat de corrolation des lignes du tableau A
Ex : [[5 0 3]		 	[[1, -0.53]
	  [3 7 9]]  donne 	 [-0.53, 1]]
1 : car corrélation entre L1 et L1, donne 1
corrélation entre L1 et L2, donne -0.53
corrélation entre L2 et L1, donne -0.53
1 : car corrélation entre L2 et L2, donne 1

np.unique(A) #renvoi tableau 1D de toutes les données triées
np.unique(A, return_counts=True) #pareil que précéd. mais renvoi un
second tableau 1D avec la répétition de chacune des données
Ex avec des binaires
A = np.random.randint(0,2,[7,7]) #mat de 0 et 1
print(np.unique(A, return_counts=True)) #affiche ([0, 1], [21, 28]

...bcp autres

"""
#print(A)

np.corrcoef(A) #mat corrélation
np.corrcoef(A)[0,1] #affiche élément ligne 0 colonne 1 de mat corrélation
np.unique(A) #affiche [0 3 5 7 9]
np.unique(A, return_counts=True) #affiche ([0, 3, 5, 7, 9],[1, 2, 1, 1, 1])
values, count = np.unique(A, return_counts = True)

for i,j in zip(values[count.argsort()], count[count.argsort()]):
	print(f'Le nombre {i} apparailt {j} fois')

print("======NUMBER CORRECTIONS======")
"""
Quand on a pas des nombres, on doit filtrer ces données pour les
manipuler.
nan = not a number
Si calcule moyenne mean() sur un tableau avec des nan, marche pas

Surtout avec Panda quer nous verrons cela !
Mais il y a sur numpy qques methodes qui travaillent avec les nans
"""

V = np.random.randn(5,5)
V[0, 0] = np.nan # 0,0 = nan
V.mean() #affiche nan = mache pas
np.nanmean(V) #affiche la moyenne sans tenir compte de nan
np.std(V) #affiche ecart type sans tenir compte de nan
np.isnan(V) #renvoie tableau bool True si nan
np.isnan(V).sum() #renvoie nb de nan
np.isnan(V).sum()/V.size #renvoie % de nan dans tableau
V[np.isnan(V)] = 0 #passe à 0 tous les nan

print("======ALGEBRE LINEAIRE======")

A = np.ones((2,3)) #mat 2l 3c
B = np.ones((3,2)) #mat 3l 2c

A.T #transposée de matrice A (échanger lignes et colonnes)
A.dot(B) #produit matriciel A.B = (2,3)*(3,2) = (2,2)
B.dot(A) #produit matriciel B.A = (3,2)*(2,3) = (3,3)

"""
package spécifique pour algèbre linéaire :
numpy.linalg

"""
A = np.random.randint(0,10,[3,3])

np.linalg.det(A) #calucl determinant
np.linalg.inv(A) #inverse la matrice A
np.linalg.pinv(A) #inverse la matrice A sans passer par combi linéaires
np.linalg.eig(A) #renvoi tableau des valeurs propres et 2nd tableau des vecteurs propres

#EXO#
"""
Standardiser un dataset très utilisé !
Permet de mettre sur une même échelle toutes les diff colonnes/lignes
Formule : (Tableau - moy(Tableau))/ecarttype
Chaque tableau corresspond à une colonne

Standardiser une mat = toutes colonnes standardisées :
veut dire que la moyenne de chaque colonne est égal à 0 and std = 1
Et dc chaque colonne suit une distribution normale parfaite
Le fait d'avoir une mat standardiser facilite les statistiques.

Standardiser la matrice A sur chaque colonne
"""
np.random.seed(0)
A = np.random.randint(0,100,[10,5])
print(A)
#il faut donc travailler sur axe 0
#On fait (10,5) - (5,)(broadcasting)
D = (A-A.mean(axis=0)) / A.std(axis = 0)
D.mean(axis = 0) # = 0
D.std(axis = 0) # = 1

print("============BROADCASTING============")
"""
Technique très puissante mais dangereuse : Broadcasting

Technique pas obligatoire, mais plus une façon dont Numpy a été conçue.
Permet de faire calculs entre tab A et tab B très rapidement.
Ex : en C, pour faire en sorte que A[i,j] + B[i,j] = C [A[i,j]+B[i,j]]
Double boucle for,... long

Avec Numpy juste C = A+B
Mais cela ne concatene pas, les valeurs a(i,j) et b(i,j) s'ajoutent
Cela ne marche que avec des matrices NUMPY de MEME SHAPE

"""
A = np.random.randint(0,10, [3,3])
B = np.random.randint(0,10, [3,3])
#K1+K2 seukement si K1.shape == K2.shape

"""
Mais il existe une technique pour "etendre les tableaux" afin de
s'additionner à tous les autres tableaux :
Le BROADCASTING = étendre les dimensions d'un tableau

Mais ne s'applique que si matrices B à une seule colonne et meme
nb de lignes. (ex : (2,3)+(2,1) = OK mais (2,3)+(2,2) = !OK)
>Mais :
Seulement, une unique colonne (n,1) et une unique ligne (1,m)
puvent se broadcaster et former une matrice (n,m) !!
Mais conséquences désastreuses car peut former des tableaux à
grandes dimensions non vouls !
Donc TOUJOURS vérifier les dimensions de matrices avec .shape, et
les redimmensionner avec .reshape()
"""
A = np.random.randint(0,10, [4,1]) #une colonne n = 4
B = np.random.randint(0,10, [1,3]) #une ligne m=3
A+B #donne matrice (4,3)

print("============RESUME NUMPY============")
"""
- Utilisé pour travailler sur tableaux à n dimensions
(en M L surtout à 2 dimlensions )

- Axe 0 : vertical et Axe 1 : horizontal (important car utile partout)

- Attribut utiles :
shape (rend nb lignes et nb colonnes du tab)
size (nb éléments)

- methodes :
concatenate(axis) (concatener avec axe 1 ou 0)
reshape((shape)) (redimensionner un tab)
ravel() (applatir tab à une seule dimension)

argsort(axis) (classement, trie dans le tableau)

- boolean indexing :
A[A<10] (très utile)

- constructeurs :
np.array(objet, dtype)
np.zeros((shape), dtype)
np.ones((shape), dtype)
np.random.randn(lignes, colonnes)
np.random.randint(min, max, (shape))

shape en []
"""


import os
import json
from copy import deepcopy

import numpy as np
import pytest

from ouster import client
from ouster.client import _client
from ouster.pcap import _pcap

from tests.conftest import PCAPS_DATA_DIR

def test_make_packets(meta: client.SensorInfo) -> None:
    pf = _client.PacketFormat.from_info(meta)

    client.ImuPacket(bytes(pf.imu_packet_size), meta)
    client.ImuPacket(bytearray(pf.imu_packet_size), meta)

    with pytest.raises(ValueError):
        client.ImuPacket(bytes(), meta)

    with pytest.raises(ValueError):
        client.ImuPacket(bytes(pf.imu_packet_size - 1), meta)

    client.LidarPacket(bytes(pf.lidar_packet_size), meta)
    client.LidarPacket(bytearray(pf.lidar_packet_size), meta)

    with pytest.raises(ValueError):
        client.LidarPacket(bytes(), meta)

    with pytest.raises(ValueError):
        client.LidarPacket(bytes(pf.lidar_packet_size - 1), meta)

def test_imu_packet(meta: client.SensorInfo) -> None:
    pf = _client.PacketFormat.from_info(meta)

    p = client.ImuPacket(bytes(pf.imu_packet_size), meta)

    assert p.sys_ts == 0
    assert p.accel_ts == 0
    assert p.gyro_ts == 0
    assert np.array_equal(p.accel, np.array([0.0, 0.0, 0.0]))
    assert np.array_equal(p.angular_vel, np.array([0.0, 0.0, 0.0]))

    with pytest.raises(AttributeError):
        p.accel_ts = 0  # type: ignore

def test_lidar_packet(meta: client.SensorInfo) -> None:
    """Test reading and writing values from empty packets."""
    pf = _client.PacketFormat.from_info(meta)
    p = client.LidarPacket(bytes(pf.lidar_packet_size), meta)
    w = pf.columns_per_packet
    h = pf.pixels_per_column

    scan_has_signal = (meta.format.udp_profile_lidar !=
                       client.UDPProfileLidar.PROFILE_LIDAR_RNG15_RFL8_NIR8)

    assert len(
        client.ChanField.__members__) == 29, "Don't forget to update tests!"
    assert np.array_equal(p.field(client.ChanField.RANGE), np.zeros((h, w)))
    assert np.array_equal(p.field(client.ChanField.REFLECTIVITY),
                          np.zeros((h, w)))
    assert np.array_equal(p.field(client.ChanField.NEAR_IR), np.zeros((h, w)))

    if scan_has_signal:
        assert np.array_equal(p.field(client.ChanField.SIGNAL), np.zeros(
            (h, w)))

    assert len(
        client.ColHeader.__members__) == 5, "Don't forget to update tests!"
    assert np.array_equal(p.header(client.ColHeader.TIMESTAMP), np.zeros(w))
    assert np.array_equal(p.timestamp, np.zeros(w))
    assert np.array_equal(p.header(client.ColHeader.FRAME_ID), np.zeros(w))
    assert np.array_equal(p.header(client.ColHeader.MEASUREMENT_ID),
                          np.zeros(w))
    assert np.array_equal(p.measurement_id, np.zeros(w))
    assert np.array_equal(p.header(client.ColHeader.ENCODER_COUNT),
                          np.zeros(w))
    assert np.array_equal(p.header(client.ColHeader.STATUS), np.zeros(w))
    assert np.array_equal(p.status, np.zeros(w))

    assert p.frame_id == 0

    # should not be able to modify packet data
    with pytest.raises(ValueError):
        p.field(client.ChanField.REFLECTIVITY)[0] = 1

    with pytest.raises(ValueError):
        p.header(client.ColHeader.MEASUREMENT_ID)[0] = 1

    with pytest.raises(ValueError):
        p.status[:] = 1

    with pytest.raises(AttributeError):
        p.frame_id = 1  # type: ignore

@pytest.mark.parametrize('test_key', ['legacy-2.0'])
def test_read_legacy_packet(packet: client.LidarPacket) -> None:
    """Read some arbitrary values from a packet and check header invariants."""
    assert packet.field(client.ChanField.RANGE)[-1, 0] == 12099
    assert packet.field(client.ChanField.REFLECTIVITY)[-1, 0] == 1017
    assert packet.field(client.ChanField.SIGNAL)[-1, 0] == 6
    assert packet.field(client.ChanField.NEAR_IR)[-1, 0] == 13

    assert np.all(np.diff(packet.header(client.ColHeader.FRAME_ID)) == 0)
    assert np.all(np.diff(packet.header(client.ColHeader.MEASUREMENT_ID)) == 1)
    assert np.all(np.diff(packet.timestamp) > 0)
    assert np.all(np.diff(packet.measurement_id) == 1)
    assert packet.packet_type == 0
    assert packet.frame_id == 5424
    assert packet.init_id == 0
    assert packet.prod_sn == 0
    assert packet.shot_limiting == 0
    assert packet.thermal_shutdown == 0
    # in 1024xN mode, the angle between measurements is exactly 88 encoder ticks
    assert np.all(np.diff(packet.header(client.ColHeader.ENCODER_COUNT)) == 88)
    assert np.all(packet.status == 0xffffffff)

@pytest.mark.parametrize('test_key', ['single-2.3'])
def test_read_single_return_packet(packet: client.LidarPacket) -> None:
    """Read some arbitrary values from packet and check header invariants."""
    assert packet.field(client.ChanField.RANGE)[-1, 0] == 11610
    assert packet.field(client.ChanField.REFLECTIVITY)[-1, 0] == 11
    assert packet.field(client.ChanField.SIGNAL)[-1, 0] == 34
    assert packet.field(client.ChanField.NEAR_IR)[-1, 0] == 393

    assert np.all(np.diff(packet.header(client.ColHeader.FRAME_ID)) == 0)
    assert np.all(np.diff(packet.header(client.ColHeader.MEASUREMENT_ID)) == 1)
    assert np.all(np.diff(packet.timestamp) > 0)
    assert np.all(np.diff(packet.measurement_id) == 1)
    assert packet.packet_type == 1
    assert packet.frame_id == 1259
    assert packet.init_id == 5431293
    assert packet.prod_sn == 992210000957
    assert packet.shot_limiting == 0
    assert packet.thermal_shutdown == 0

    # Changes from LEGACY
    assert np.all(np.diff(packet.header(client.ColHeader.ENCODER_COUNT)) == 0)
    assert np.all(packet.status == 0x01)

def test_scan_writeable() -> None:
    """Check that a native scan is a writeable view of data."""
    ls = client.LidarScan(1024, 32)

    assert not ls.field(client.ChanField.RANGE).flags.owndata
    assert not ls.status.flags.owndata

    assert ls.field(client.ChanField.SIGNAL).flags.aligned
    assert ls.measurement_id.flags.aligned

    assert ls.field(client.ChanField.NEAR_IR).flags.aligned
    assert ls.timestamp.flags.aligned

    ls.field(client.ChanField.RANGE)[0, 0] = 42
    assert ls.field(client.ChanField.RANGE)[0, 0] == 42

    ls.field(client.ChanField.RANGE)[:] = 7
    assert np.all(ls.field(client.ChanField.RANGE) == 7)

    ls.status[-1] = 0xffff
    assert ls.status[-1] == 0xffff

    ls.status[:] = 0x1
    assert np.all(ls.status == 0x1)

    assert np.all(ls.pose == np.eye(4))

    ls.pose[1][0, 2] = 8
    assert np.all(ls.pose[1] == np.array([[1, 0, 8, 0], [0, 1, 0, 0],
                                          [0, 0, 1, 0], [0, 0, 0, 1]]))

def test_scan_from_native() -> None:
    ls = client.LidarScan(1024, 32)
    ls2 = client.LidarScan.from_native(ls)

    assert ls is ls2

def test_scan_to_native() -> None:
    ls = client.LidarScan(1024, 32)
    ls2 = ls.to_native()

    assert ls is ls2

def test_scan_field_ref() -> None:
    """Test that field references keep scans alive."""

    ls = client.LidarScan(512, 16)
    range = ls.field(client.ChanField.RANGE)
    range[:] = 42

    del ls
    assert np.all(range == 42)

    range[:] = 43
    assert np.all(range == 43)

def test_scan_header_ref() -> None:
    """Test that header references keep scans alive."""

    ls = client.LidarScan(512, 16)
    status = ls.status
    status[:] = 0x11

    del ls
    assert np.all(status == 0x11)

    status[:] = 0x01
    assert np.all(status == 0x01)

def test_scan_not_complete() -> None:
    """Test that not all scans are considered complete."""
    ls = client.LidarScan(32, 1024)

    status = ls.status
    assert not ls.complete()

    status[0] = 0x02
    assert not ls.complete()
    assert not ls.complete((0, 0))

    status[1:] = 0xFFFFFFFF
    assert not ls.complete()

    status[:] = 0xFFFFFFFF
    status[-1] = 0x02
    assert not ls.complete()

    # windows are inclusive but python slicing is not
    status[:] = 0x00
    status[:10] = 0xFFFFFFFF
    assert not ls.complete((0, 10))

    status[:] = 0x00
    status[11:21] = 0xFFFFFFFF
    assert not ls.complete((10, 20))

    # window [i, i]
    status[:] = 0x00
    status[0] = 0xFFFFFFFF
    assert not ls.complete()
    assert not ls.complete((0, 1))
    assert ls.complete((0, 0))

    status[:] = 0x00
    status[128] = 0xFFFFFFFF
    assert not ls.complete()
    assert not ls.complete((127, 128))
    assert ls.complete((128, 128))

# TODO: Add 4096 to this test
@pytest.mark.parametrize("w, win_start, win_end", [
    (512, 0, 511),
    (512, 1, 0),
    (512, 256, 0),
    (512, 256, 1),
    (1024, 0, 1023),
    (1024, 0, 512),
    (1024, 0, 0),
    (1024, 1023, 1023),
    (1024, 1023, 0),
    (1024, 1023, 1),
    (2048, 0, 2047),
    (2048, 1024, 512),
    (2048, 1024, 0),
    (2048, 1024, 1),
    (2048, 511, 511),
])
def test_scan_complete(w, win_start, win_end) -> None:
    """Set the status headers to the specified window and check complete()."""
    ls = client.LidarScan(32, w)

    status = ls.status

    if win_start <= win_end:
        status[win_start:win_end + 1] = 0xFFFFFFFF
    else:
        status[0:win_end + 1] = 0xFFFFFFFF
        status[win_start:] = 0xFFFFFFFF

    assert ls.complete((win_start, win_end))

def test_scan_fields_ref() -> None:
    """Make sure ref to fields keeps scan alive."""
    fields = client.LidarScan(32, 1024).fields

    # should fail (or trip asan) if the field iterator doesn't keep scan alive
    assert set(fields) == {
        client.ChanField.RANGE,
        client.ChanField.REFLECTIVITY,
        client.ChanField.SIGNAL,
        client.ChanField.NEAR_IR,
    }

def test_scan_default_fields() -> None:
    """Default scan has the expected fields for the LEGACY profile."""
    ls = client.LidarScan(32, 1024)

    assert set(ls.fields) == {
        client.ChanField.RANGE,
        client.ChanField.REFLECTIVITY,
        client.ChanField.SIGNAL,
        client.ChanField.NEAR_IR,
    }

    for f in ls.fields:
        assert ls.field(f).dtype == np.uint32

def test_scan_dual_profile() -> None:
    """Dual returns scan has the expected fields."""
    ls = client.LidarScan(
        32, 1024,
        client.UDPProfileLidar.PROFILE_LIDAR_RNG19_RFL8_SIG16_NIR16_DUAL)

    assert set(ls.fields) == {
        client.ChanField.RANGE,
        client.ChanField.RANGE2,
        client.ChanField.REFLECTIVITY,
        client.ChanField.REFLECTIVITY2,
        client.ChanField.SIGNAL,
        client.ChanField.SIGNAL2,
        client.ChanField.NEAR_IR,
    }

def test_scan_low_data_rate() -> None:
    """Low Data Rate scan has the expected fields."""
    ls = client.LidarScan(32, 1024,
                          client.UDPProfileLidar.PROFILE_LIDAR_RNG15_RFL8_NIR8)

    assert set(ls.fields) == {
        client.ChanField.RANGE,
        client.ChanField.REFLECTIVITY,
        client.ChanField.NEAR_IR,
    }

def test_scan_single_return() -> None:
    """Single Return scan has the expected fields."""
    ls = client.LidarScan(
        32, 1024, client.UDPProfileLidar.PROFILE_LIDAR_RNG19_RFL8_SIG16_NIR16)

    assert set(ls.fields) == {
        client.ChanField.RANGE,
        client.ChanField.REFLECTIVITY,
        client.ChanField.SIGNAL,
        client.ChanField.NEAR_IR,
    }

def test_scan_empty() -> None:
    """Sanity check scan with no fields."""
    ls = client.LidarScan(32, 1024, {})

    assert set(ls.fields) == set()

    for f in client.ChanField.values:
        with pytest.raises(ValueError):
            ls.field(f)

def test_scan_custom() -> None:
    """Sanity check scan with a custom set of fields."""
    ls = client.LidarScan(
        32, 1024, {
            client.ChanField.SIGNAL: np.uint16,
            client.ChanField.FLAGS: np.uint8,
            client.ChanField.CUSTOM0: np.uint32
        })

    assert set(ls.fields) == {
        client.ChanField.SIGNAL, client.ChanField.FLAGS,
        client.ChanField.CUSTOM0
    }
    assert ls.field(client.ChanField.SIGNAL).dtype == np.uint16
    assert ls.field(client.ChanField.CUSTOM0).dtype == np.uint32

    with pytest.raises(ValueError):
        ls.field(client.ChanField.RANGE)

def test_scan_eq_fields() -> None:
    """Test equality between scans with different fields."""
    ls0 = client.LidarScan(32, 1024)
    ls1 = client.LidarScan(32, 1024,
                           client.UDPProfileLidar.PROFILE_LIDAR_LEGACY)
    ls2 = client.LidarScan(
        32, 1024,
        client.UDPProfileLidar.PROFILE_LIDAR_RNG19_RFL8_SIG16_NIR16_DUAL)
    ls3 = client.LidarScan(32, 1024, {client.ChanField.SIGNAL: np.uint32})
    ls4 = client.LidarScan(32, 1024, {client.ChanField.SIGNAL: np.uint16})
    ls5 = client.LidarScan(32, 1024, {})

    assert ls0 == ls1
    assert not (ls0 != ls1)  # should be implemented using __eq__
    assert ls1 != ls2
    assert ls3 != ls4
    assert ls5 != ls0
    assert ls5 != ls2
    assert ls5 != ls4

def test_scan_zero_init() -> None:
    """Test that scan fields and headers are zero initialized."""
    ls = client.LidarScan(
        64, 1024,
        client.UDPProfileLidar.PROFILE_LIDAR_RNG19_RFL8_SIG16_NIR16_DUAL)

    assert ls.frame_id == -1
    assert ls.frame_status == 0

    assert np.count_nonzero(ls.timestamp) == 0
    assert np.count_nonzero(ls.measurement_id) == 0
    assert np.count_nonzero(ls.status) == 0

    for f in ls.fields:
        assert np.count_nonzero(ls.field(f)) == 0

def test_scan_copy_eq() -> None:
    """Test equality with a copy."""

    ls0 = client.LidarScan(32, 512)
    ls0.status[:] = 0x1
    ls0.field(client.ChanField.REFLECTIVITY)[:] = 100
    ls0.pose[:, 0, 3] = 8

    ls1 = deepcopy(ls0)

    assert ls0 is not ls1
    assert ls0 == ls1

    ls0.frame_id = 9
    assert ls0 != ls1

    ls1.frame_id = 9
    assert ls0 == ls1

    ls0.frame_status = 1
    assert ls0 != ls1

    ls1.frame_status = 1
    assert ls0 == ls1

    ls0.measurement_id[0] = 1
    assert ls0 != ls1

    ls1.measurement_id[0] = 1
    assert ls0 == ls1

    ls0.field(client.ChanField.RANGE)[0, 0] = 42
    assert ls0 != ls1

    ls1.field(client.ChanField.RANGE)[0, 0] = 42
    assert ls0 == ls1

    ls0.pose[1] = np.eye(4)
    assert ls0 != ls1

    ls0.pose[1, 0, 3] = 8
    assert ls0 == ls1

def test_scan_eq_with_custom_fields() -> None:
    """Test equality with custom fields."""

    ls0 = client.LidarScan(32, 512, {
        client.ChanField.CUSTOM0: np.uint32,
        client.ChanField.CUSTOM4: np.uint8
    })

    ls1 = deepcopy(ls0)

    ls0.field(client.ChanField.CUSTOM0)[:] = 100

    ls2 = deepcopy(ls0)

    assert np.count_nonzero(
        ls2.field(client.ChanField.CUSTOM0) == 100) == ls0.h * ls0.w
    assert np.count_nonzero(ls2.field(client.ChanField.CUSTOM4) == 100) == 0

    assert ls1 is not ls0
    assert ls1 != ls0
    assert ls2 == ls0

def test_error_eq() -> None:
    assert client.PacketSizeError("abc") == client.PacketSizeError("abc")

def test_lidar_packet_validator() -> None:
    """LidarPacketValidator should be capable of the same init_id/sn check as LidarPacket."""
    meta_file_path = os.path.join(PCAPS_DATA_DIR, 'OS-0-128-U1_v2.3.0_1024x10.json')
    pcap_file_path = os.path.join(PCAPS_DATA_DIR, 'OS-0-128-U1_v2.3.0_1024x10.pcap')
    metadata = client.SensorInfo(open(meta_file_path).read())

    metadata2 = deepcopy(metadata)
    metadata2.init_id = 1234
    metadata2.sn = "5678"

    validator = client.LidarPacketValidator(metadata2)
    pcap_handle = _pcap.replay_initialize(pcap_file_path)
    buf = bytearray(2**16)
    info = _pcap.packet_info()
    _pcap.next_packet_info(pcap_handle, info)
    n_bytes = _pcap.read_packet(pcap_handle, buf)
    errors = validator.check_packet(buf, n_bytes)
    assert n_bytes == 8448
    try:
        # LidarPacket constructor should throw the same error if _raise_on_id_check is True
        client.LidarPacket(buf, metadata2, None, _raise_on_id_check=True)
    except client.data.PacketIdError as e:
        assert type(errors) == list
        assert len(errors) == 1
        assert str(e) == str(errors[0])
        assert str(e) == "Metadata init_id/sn does not match: expected by metadata \
- 1234/5678, but got from packet buffer - 5431292/122150000150"
        pass
    else:
        assert False, "Expected an exception to be raised by client.LidarPacket."
    finally:
        _pcap.replay_uninitialize(pcap_handle)

def test_lidar_packet_validator_2() -> None:
    """LidarPacketValidator check_packet should return None if there are no issues."""
    meta_file_path = os.path.join(PCAPS_DATA_DIR, 'OS-0-128-U1_v2.3.0_1024x10.json')
    pcap_file_path = os.path.join(PCAPS_DATA_DIR, 'OS-0-128-U1_v2.3.0_1024x10.pcap')
    metadata = client.SensorInfo(open(meta_file_path).read())

    metadata2 = deepcopy(metadata)

    validator = client.LidarPacketValidator(metadata2)
    pcap_handle = _pcap.replay_initialize(pcap_file_path)
    buf = bytearray(2**16)
    info = _pcap.packet_info()
    _pcap.next_packet_info(pcap_handle, info)
    n_bytes = _pcap.read_packet(pcap_handle, buf)
    assert n_bytes == 8448
    assert validator.check_packet(buf, n_bytes) == []
    _pcap.replay_uninitialize(pcap_handle)

def test_lidar_packet_validator_3() -> None:
    """LidarPacketValidator check_packet should identify a packet
    that's the wrong size according to the lidar UDP profile."""
    meta_file_path = os.path.join(PCAPS_DATA_DIR, 'OS-0-128-U1_v2.3.0_1024x10.json')
    pcap_file_path = os.path.join(PCAPS_DATA_DIR, 'OS-0-128-U1_v2.3.0_1024x10.pcap')
    metadata = client.SensorInfo(open(meta_file_path).read())

    metadata2 = deepcopy(metadata)
    metadata2.format.udp_profile_lidar = client.UDPProfileLidar.PROFILE_LIDAR_FIVE_WORD_PIXEL

    validator = client.LidarPacketValidator(metadata2)
    pcap_handle = _pcap.replay_initialize(pcap_file_path)
    buf = bytearray(2**16)
    info = _pcap.packet_info()
    _pcap.next_packet_info(pcap_handle, info)
    n_bytes = _pcap.read_packet(pcap_handle, buf)
    assert n_bytes == 8448
    errors = validator.check_packet(buf, n_bytes)
    assert len(errors) == 1
    assert type(errors[0]) == client.PacketSizeError
    assert str(errors[0]) == 'Expected a packet of size 41216 but got a buffer of size 8448'
    _pcap.replay_uninitialize(pcap_handle)

def test_lidar_packet_validator_4() -> None:
    """LidarPacketValidator check_packet should identify a packet
    that's the wrong size according to the data format."""
    meta_file_path = os.path.join(PCAPS_DATA_DIR, 'OS-0-128-U1_v2.3.0_1024x10.json')
    pcap_file_path = os.path.join(PCAPS_DATA_DIR, 'OS-0-128-U1_v2.3.0_1024x10.pcap')
    meta_json = json.load(open(meta_file_path))
    meta_json['data_format']['columns_per_frame'] = 4096
    meta_json['data_format']['pixels_per_column'] = 16
    meta_json['data_format']['pixel_shift_by_row'] = [0] * 16
    meta_json['beam_altitude_angles'] = [1] * 16
    meta_json['beam_azimuth_angles'] = [1] * 16
    meta_json['prod_sn'] = '1234'
    metadata2 = client.SensorInfo(json.dumps(meta_json))

    validator = client.LidarPacketValidator(metadata2)
    pcap_handle = _pcap.replay_initialize(pcap_file_path)
    buf = bytearray(2**16)
    info = _pcap.packet_info()
    _pcap.next_packet_info(pcap_handle, info)
    n_bytes = _pcap.read_packet(pcap_handle, buf)
    assert n_bytes == 8448
    errors = validator.check_packet(buf, n_bytes)
    assert len(errors) == 2
    assert type(errors[0]) == client.PacketIdError
    assert str(
        errors[0]) == 'Metadata init_id/sn does not match: expected by metadata - \
5431292/1234, but got from packet buffer - 5431292/122150000150'
    assert type(errors[1]) == client.PacketSizeError
    assert str(errors[1]) == 'Expected a packet of size 1280 but got a buffer of size 8448'
    _pcap.replay_uninitialize(pcap_handle)


#!/usr/bin/env python
# -*- coding: utf-8; -*-

import urllib

import httplib2
from multipart import encode_multipart, BOUNDARY

class Http(httplib2.Http):
    """
    Subclass of Http that supports multipart style file uploading via POST as
    and cookies.
    """
    def request_with_files(self, url, method, body=None, body_files=None,
                           headers=None, *args, **kwargs):
        """
        Note: Unlike the standard request method, the body & body_files params
        to this method should *not* be urlencoded strings. They should instead
        be urlencodable objects (e.g. a dict), and will be urlencoded
        automatically by this method after the body files are processed.
        """
        if body_files:
            body = encode_multipart(body, body_files)
            headers["Content-type"] = \
                    "multipart/form-data; boundary=%s" % BOUNDARY
            headers["Content-length"] = str(len(body))
        else:
            body = urllib.urlencode(body)
            headers.setdefault("Content-type",
                               "application/x-www-form-urlencoded")

        return super(Http, self).request(url, method, body,
                                         headers, *args, **kwargs)


from __future__ import annotations

from speechkit import Session, SpeechSynthesis, DataStreamingRecognition
from decouple import config
from io import BytesIO
from pyaudio import PyAudio, paInt16

class Speecher(object):

    CHANNELS_NUMBER = 1
    SAMPLE_RATE = 8_000
    CHUNK_SIZE = 4_000

    VOICE = "ermil"
    EMOTION = "good"
    AUDIO_FORMAT = "lpcm"
    LANGUAGE_CODE = "ru-RU"
    AUDIO_ENCODING = "LINEAR16_PCM"

    __oauth_token: str
    __catalog_id: str
    __session: Session

    def __init__(self: Speecher) -> None:
        self.__oauth_token = config('oauth_token', default='')
        self.__catalog_id = config('catalog_id', default='')
        self.__session = Session.from_yandex_passport_oauth_token(
            self.__oauth_token,
            self.__catalog_id
        )

    def __pyaudio_play_audio_function(
        self: Speecher,
        audio_data: BytesIO
    ) -> None:

        pyaudio_object = PyAudio()
        stream = pyaudio_object.open(
            format              = paInt16,
            channels            = self.CHANNELS_NUMBER,
            rate                = self.SAMPLE_RATE,
            output              = True,
            frames_per_buffer   = self.CHUNK_SIZE
        )
        for i in range(0, len(audio_data), self.CHUNK_SIZE):
            stream.write(audio_data[i:i + self.CHUNK_SIZE])

        stream.stop_stream()
        stream.close()
        pyaudio_object.terminate()

    def synthesize(self: Speecher, input_text: str) -> None:
        synthesizeAudio = SpeechSynthesis(self.__session)
        audio_data = synthesizeAudio.synthesize_stream(
            text            = input_text,
            voice           = self.VOICE,
            emotion         = self.EMOTION,
            format          = self.AUDIO_FORMAT,
            sampleRateHertz = self.SAMPLE_RATE
        )
        self.__pyaudio_play_audio_function(audio_data)

    def __gen_audio_capture_function(self: Speecher):
        pyaudio_object = PyAudio()
        stream = pyaudio_object.open(
            format              = paInt16,
            channels            = self.CHANNELS_NUMBER,
            rate                = self.SAMPLE_RATE,
            input               = True,
            frames_per_buffer   = self.CHUNK_SIZE
        )
        try:
            while True:
                yield stream.read(self.CHUNK_SIZE)
        finally:
            stream.stop_stream()
            stream.close()
            pyaudio_object.terminate()

    def recognize(self: Speecher) -> str:
        data_streaming_recognition = DataStreamingRecognition(
            self.__session,
            language_code       = self.LANGUAGE_CODE,
            audio_encoding      = self.AUDIO_ENCODING,
            sample_rate_hertz   = self.SAMPLE_RATE,
            partial_results     = False,
            single_utterance    = True,
        )

        result_data = ""
        for text, final, end_of_utt in data_streaming_recognition.recognize(
            self.__gen_audio_capture_function
        ):
            result_data += text[0]
            if final:
                break

        return result_data


import random
from typing import Tuple

import numpy as np
import torch
from torch import nn

from src.models.LLFlow.flow_layers import SqueezeLayer  # noqa: I900

class Interpolate(nn.Module):
    def __init__(
        self,
        scale_factor,
        mode="nearest",
        recompute_scale_factor=True,
    ):
        super(Interpolate, self).__init__()
        self.func = nn.functional.interpolate
        self.scale_factor = scale_factor
        self.mode = mode
        self.recompute_scale_factor = recompute_scale_factor

    def forward(self, x):
        return self.func(
            x,
            scale_factor=self.scale_factor,
            mode=self.mode,
            recompute_scale_factor=self.recompute_scale_factor,
        )

class Conv2dZeros(nn.Conv2d):
    def __init__(
        self,
        in_channels,
        out_channels,
        kernel_size=(3, 3),
        stride=1,
        padding=0,
        logscale_factor=3,
    ):
        super().__init__(in_channels, out_channels, kernel_size, stride, padding)

        self.logscale_factor = logscale_factor
        self.register_parameter("logs", nn.Parameter(torch.zeros(out_channels, 1, 1)))

        self.weight.data.zero_()
        self.bias.data.zero_()

    def forward(self, x):
        output = super().forward(x)
        return output * torch.exp(self.logs * self.logscale_factor)

class LLFlowNLL:
    Log2PI = float(np.log(2 * np.pi))

    def __init__(self, p=0.2, random_state=42):
        random.seed(random_state)
        self.p = p
        self.squeeze = SqueezeLayer(factor=8)

    def __call__(self, z, logdet, encoded_color_map, gt):
        reference = (
            encoded_color_map if random.random() > self.p else LLFlowNLL.color_map(gt)
        )
        reference, _ = self.squeeze(reference)

        objective = logdet + LLFlowNLL.log_prob(
            z, mean=reference, log_var=torch.tensor(0.0).to(z.device)
        )
        return -objective / float(np.log(2.0) * gt.size(2) * gt.size(3))

    @staticmethod
    def color_map(image: torch.tensor) -> torch.tensor:
        return image / (torch.sum(image, dim=1, keepdim=True) + 1e-4)

    @staticmethod
    def log_prob(x, mean, log_var):
        return torch.sum(LLFlowNLL.likelihood(x, mean, log_var), dim=(1, 2, 3))

    @staticmethod
    def likelihood(x, mean, log_var):
        return -0.5 * (
            log_var * 2.0
            + ((x - mean) ** 2) / torch.exp(log_var * 2.0)
            + LLFlowNLL.Log2PI
        )

def split_channels(z: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
    return z[:, ::2, ...], z[:, 1::2, ...]


import streamlit as st
import pandas as pd
import numpy as np
from feature_description import DISPLAY_FEATURE_DESCRIPTION, load_data
import pickle
from sklearn.preprocessing import MinMaxScaler
from sklearn.svm import SVR

#adding local image
image_path = "house_image.jpg"

# Display the image with a caption
st.image(image_path, caption="My Great Asset", use_column_width=200)

# Set the header color
st.header(":red[Boston House Price Prediction]")

# Load the data
data = pd.read_csv('boston_housing.csv')

# Adding feature description to sidebar
st.sidebar.subheader("**:blue[Check features description here.]**")
st.sidebar.write('Select feature')
feature = st.sidebar.selectbox('Select feature', options=['CRIM', 'INDUS', 'NOX', 'RM', 'AGE', 'DIS', 'TAX', 'PTRATIO', 'LSTAT'], label_visibility='collapsed')
if st.sidebar.button('**:red[Show feature description]**'):
    DISPLAY_FEATURE_DESCRIPTION(feature)

# Define default values for input fields
default_CRIM = 0.0
default_INDUS = 0.0
default_NOX = 0.0
default_RM = 6.0
default_AGE = 0.0
default_DIS = 0.0
default_TAX = 187.0
default_PTRATIO = 12.0
default_LSTAT = 0.0

# Setting input parameters
left_column, middle_column, right_column = st.columns(3)

CRIM = left_column.number_input('CRIM', min_value=0.000, max_value=100.000, step=0.001, value=default_CRIM)
INDUS = middle_column.number_input('INDUS', min_value=0.000, max_value=30.000, step=0.001, value=default_INDUS)
NOX = right_column.number_input('NOX', min_value=0.000, max_value=1.000, step=0.001, value=default_NOX)

RM = left_column.number_input('RM', min_value=3.000, max_value=9.000, step=0.001, value=default_RM)
AGE = middle_column.number_input('AGE', min_value=0.000, max_value=100.000, step=0.001, value=default_AGE)
DIS = right_column.number_input('DIS', min_value=0.000, max_value=13.000, step=0.001, value=default_DIS)

TAX = left_column.number_input('TAX', min_value=187.0, max_value=711.0, step=0.1, value=default_TAX)
PTRATIO = middle_column.number_input('PTRATIO', min_value=12.00, max_value=22.00, step=0.01, value=default_PTRATIO)
LSTAT = right_column.number_input('LSTAT', min_value=0.00, max_value=40.00, step=0.01, value=default_LSTAT)

# Loading the model
with open('price_pred_model.sav', 'rb') as ppm:
    model = pickle.load(ppm)

# Creating a dataframe of input features to make it 2-dimensional
input_features = {"CRIM": CRIM, "INDUS": INDUS, "NOX": NOX, "RM": RM, "AGE": AGE, "DIS": DIS, "TAX": TAX, "PTRATIO": PTRATIO, "LSTAT": LSTAT}
input_features_frame = pd.DataFrame(input_features, index=[0])

# Feature scaling
scaler = MinMaxScaler()
input_scaled = scaler.fit_transform(input_features_frame)

# Verify Input Features
st.write("**:red[Verify Input Features]**")
st.write(input_features_frame)

# Predicting
predicted_price = model.predict(input_scaled)

# Printing prediction
if st.button('Show predicted price', key='show_predicted', help='Click to see the predicted price'):
    st.subheader(f":blue[The predicted price of the house is between USD {np.round((predicted_price[0] - 2.8) * 1000)} and USD {np.round((predicted_price[0] + 2.8) * 1000)}]")

# Add a button for making another prediction
if st.button('Make another prediction', key='make_another', help='Click to make another prediction'):
    # Reset the input fields to their default values
   # Add a button for making another prediction
    CRIM = 0.0
    INDUS = 0.0
    NOX = 0.0
    RM = 6.0
    AGE = 0.0
    DIS = 0.0
    TAX = 187.0
    PTRATIO = 12.0
    LSTAT = 0.0


import datetime
import time
from argparse import ArgumentParser

from pytomato.argument_parser import setupParser
from pytomato.conf import (BREAK_TYPE, DEFAULT_BREAK_NAME, DEFAULT_TOMATO_NAME,
                           LONG_BREAK_DURATION, LONG_TOMATO_DURATION,
                           SHORT_BREAK_DURATION, SHORT_TOMATO_DURATION,
                           TOMATO_TYPE)
from pytomato.run_parameters import RunParameters
from pytomato.utility import formatToHHMM

def _set_up_break_or_tomato(parameters: RunParameters, args: ArgumentParser,
                            short_duration: int, long_duration: int, default_name: str, run_type: str) -> RunParameters:
    """
    :param args: The argument parser
    :param short_dur: Short duration for the run type
    :param long_dur: Long duration for the run type
    :param custom_dur: Custom duration provided by the user
    :param custom_name: Custom name provided by the user
    :param run_type: The run type
    """
    if args.short_break:  # type: ignore
        parameters.duration = short_duration

    if args.long_break:  # type: ignore
        parameters.duration = long_duration

    if args.duration:  # type: ignore
        parameters.duration = args.duration  # type: ignore

    if args.name:  # type: ignore
        parameters.run_name = args.name  # type: ignore
    else:
        parameters.run_name = default_name

    parameters.run_type = run_type

    return parameters

def set_up_run_parameters(parameters: RunParameters, args: ArgumentParser) -> RunParameters:
    # can't do both at the same time, we prefer long
    if args.short and args.long:  # type: ignore
        args.short = False  # type: ignore

    # run without any parameters it will be a short run
    if not args.duration and not args.short and not args.long:  # type: ignore
        args.short = True  # type: ignore

    if args.short_break or args.long_break:  # type: ignore
        parameters = _set_up_break_or_tomato(parameters, args, SHORT_BREAK_DURATION,
                                             LONG_BREAK_DURATION, DEFAULT_BREAK_NAME, BREAK_TYPE)

    else:
        parameters = _set_up_break_or_tomato(parameters, args, SHORT_TOMATO_DURATION,
                                             LONG_TOMATO_DURATION, DEFAULT_TOMATO_NAME, TOMATO_TYPE)

    if args.project:  # type: ignore
        # this will have the default value from conf.py if not changed here
        parameters.project_name = args.project  # type: ignore

    parameters.clean = args.clean  # type: ignore
    parameters.list_and_exit = args.list  # type: ignore
    parameters.delete = args.delete  # type: ignore
    parameters.mute = args.mute  # type: ignore
    parameters.force_upload = args.upload  # type: ignore

    return parameters

def main():
    parser = setupParser()
    args = parser.parse_args()

    parameters = set_up_run_parameters(RunParameters(), args)

    if args.cli:  # type: ignore
        from pytomato.timer import Timer
        timer = Timer(parameters)
    else:
        try:
            from pytomato.guitimer import GUITimer
            timer = GUITimer(parameters)
        except ImportError as exc:
            print("Could not create GUI timer, you need PyQt5 installed! Or run with --cli for console mode.")
            return

    run(timer, parameters)

def run(timer, parameters):
    if parameters.clean:
        timer.clean()
        return

    target_time = parameters.duration
    timer.init(parameters.run_type, target_time)
    timer.list_entries()

    if parameters.list_and_exit:
        return

    # check if not none, because single integers get cast to booleans..
    if parameters.delete is not None:
        timer.delete(parameters.delete)
        return

    if parameters.force_upload:
        # The parameter is set on initialisation, we only need to call save and it will be upload
        timer.force_upload()
        return

    print("Project:", parameters.project_name)
    print("Entry name:", parameters.run_name)
    print("Target time", target_time // 60, "minutes")

    start_datetime = datetime.datetime.now()
    elapsed_time = 0
    user_notified = False

    target_timeString = formatToHHMM(target_time)
    # start the initial update timer
    update_compensation_timer = time.time()
    try:
        while True:
            timer.updateVisuals(elapsed_time, target_time, target_timeString)
            # sleep for the rest of the 1 second, by subtracting the time spent processing
            time.sleep(abs(1 - (time.time() - update_compensation_timer)))
            elapsed_time += 1

            # warn the user only when we're over the target time and we haven't previously notified them
            if not user_notified and elapsed_time >= target_time:
                timer.notify()
                user_notified = True
            # update the update timer after everything has been completed timer
            update_compensation_timer = time.time()

    # if anything happens try to end and save out the file
    # note: the strings below are intended to have empty spaces, to make sure they overwrite the whole line
    # because the print in the while loop uses \r to rewrite the line
    except KeyboardInterrupt:
        print("Interripting timer and saving entry.                  ")
    except SystemExit:
        print("System exiting, saving entry.                         ")

    end_datetime = datetime.datetime.now()
    timer.closeEvent()
    timer.process_entry(start_datetime, end_datetime, elapsed_time, target_time)


from time import sleep
from random import randint
import os

while True:
        sleep_sec = 1
        sleep(sleep_sec)
        print(f'prng is running')
        with open('prng-service.txt', 'r', encoding="utf-8") as f:
                read_data = f.read()

        if read_data == "run":
                with open('prng-service.txt', 'w', encoding="utf-8") as f:
                        num_files = len(os.listdir("./cs361"))
                        random_file_num = randint(1, num_files)
                        print(f'Putting random number into prng-service.txt')
                        f.write(str(random_file_num))


# -*- coding: cp1252 -*-
#####################
# Ce script créé un fichier de particules qui forme une sinusoïde
#
#####################

import libsimpa as ls
from math import *

pwriter=ls.ParticuleIO()
pwriter.OpenForWrite("particules.pbin")
pwriter.SetHeaderData(.02)
nbpart=10000
nbtimestep=12
pospart=[[i/800.,2,1+pi/100.*(i/10.)] for i in range(0,nbpart)]

for part in pospart:
    pwriter.NewParticle(0) #pas de temps de départ n°0
    for i in range(0,nbtimestep):
        pwriter.NewPositionParticle(part[0],2,(sin(part[2]+(i*(pi/6)))+2),1.)
pwriter.Close()


#Maria Jose Castro 181202
#Roberto Figueroa 18306
#Diana de Leon 18607

#Ejercicio 1
#paquetes necesarios
import math
import itertools

def genElements(n):
    return list(range(n))

mensaje = "1.Orden-Sin reemplazo\n2.Orden-Con remplazo\n3.Sin orden-sin remplazo\n4.Sin orden-con remplazo\n5.Salir"
opcion = 0
opcion= int(input("\nMenu \n"+ mensaje + "\n" + "Ingrese una opcion del menu: "))
while opcion != 5:
    n = int(input("\nIngrese el numero de elementos: "))
    r = int(input("\nIngrese el numero de elementos de la muestra: "))

    resultado = 0
    perm = []
    if opcion == 1:
        resultado = math.factorial(n) / math.factorial((n-r))
        perm = itertools.permutations(genElements(n),r)
    elif opcion == 2:
        resultado = n ** r
        perm = itertools.product(genElements(n),genElements(n))
    elif opcion == 3:
        resultado = math.factorial(n) / (math.factorial(r)* math.factorial((n-r)))
        perm = itertools.combinations(genElements(n),r)
    elif opcion == 4:
        resultado = math.factorial((r+n-1)) / (math.factorial(r)* math.factorial((n-1)))
        perm = itertools.combinations_with_replacement(genElements(n),r)
    else:
        print("Opcion no valida")
    print("La cantidad es : " + str(int(resultado)))
    print("Las combinaciones/permutaciones son: ")
    print(list(perm))

    opcion= int(input("\nMenu \n"+ mensaje + "\n" + "Ingrese una opcion del menu: "))


class AdjNode:
    def __init__(self, value):
        self.value = value
        self.next = None

class AdjListDigraph:

    def __init__(self, num_vertices):
        self.storage = [None] * num_vertices
        self.num_vertices = num_vertices

    def add_edge(self, v, w):
        node = AdjNode(w)
        node.next = self.storage[v]
        self.storage[v] = node

    def remove_edge(self, v, w):
        if self.storage[v] is None:
            raise Exception('No adjacent vertices present for {}'.format(v))

        node = self.storage[v]
        if node.value == w:
            self.storage[v] = node.next
            return

        previous_node = node
        while(node is not None):
            if node.value == w:
                previous_node.next = node.next
                return

            previous_node = node
            node = node.next

        raise Exception('No adjacent vertex {} present for {}'.format(w, v))

    def adjacent(self, v):
        adjacent_vertices = []
        node = self.storage[v]

        while(node is not None):
            adjacent_vertices.append(node.value)
            node = node.next

        return adjacent_vertices

    def num_of_vertices(self):
        return self.num_vertices

    def num_of_edges(self):
        num_edges = 0
        for v in range(self.num_vertices):
            for _ in self.adjacent(v):
                num_edges += 1

        return num_edges

    def out_degree(self, v):
        count = 0
        for _ in self.adjacent(v):
            count += 1

        return count

    def max_out_degree(self):
        max = 0
        for v in range(self.num_vertices):
            d = self.out_degree(v)
            if d > max:
                max = d

        return max

    def average_out_degree(self):
        return self.num_of_edges() / self.num_of_vertices()

    def num_of_self_loops(self):
        count = 0
        for v in range(self.num_vertices):
            adjacent_vertices = self.adjacent(v)
            if v in adjacent_vertices:
                count += 1

        return count

def create_digraph_one(digraph):
    for v, w in [(0, 1), (0, 5), (2, 0), (2, 3), (3, 2), (3, 5), (4, 2), (4, 3),
                (5, 4), (6, 0), (6, 4), (6, 8), (6, 9), (7, 6), (7, 9), (8, 6),
                (9, 10), (9, 11), (10, 12), (11, 4), (11, 12), (12, 9)]:
        digraph.add_edge(v, w)

    return digraph

def create_digraph_two(digraph):
    for v, w in [(0, 1), (0, 2), (0, 5), (1, 4), (3, 2), (3, 4),
                (3, 5), (3, 6), (5, 2), (6, 0), (6, 4)]:
        digraph.add_edge(v, w)

    return digraph

if __name__ == '__main__':
    digraph_one = AdjListDigraph(13)
    digraph_one = create_digraph_one(digraph_one)

    for v in [9, 4, 12]:
        print('Following are the vertices adjacent to {}'.format(v))
        print(digraph_one.adjacent(v))

    print('Number of vertices in the digraph: {}'.format(digraph_one.num_of_vertices()))
    print('Number of edges in the digraph: {}'.format(digraph_one.num_of_edges()))
    print('Max Degree: {}'.format(digraph_one.max_out_degree()))
    print('Average Degree: {}'.format(digraph_one.average_out_degree()))
    print('No. of self loops: {}'.format(digraph_one.num_of_self_loops()))


from django.shortcuts import render
from django.http import HttpResponse
from django.shortcuts import get_object_or_404, render, redirect
from .forms import importdataform
# Create your views here.
def importData (request):
    if request.method == 'POST':
        dataset = request.POST['dataset']
        import_file = request.FILES['importData']
        if (str(import_file).split('.')[-1] == "xlsx"):

            logger.error("You are Here in Household Section Section '{0}'".format(import_file))

            sheet = import_file.get_records()
            TR = len(sheet)
            i = 0
            j = 0
            iniciate = datetime.utcnow().replace(tzinfo=utc)
            logger.error(iniciate)
            for record in sheet:
                Rtime = datetime.utcnow().replace(tzinfo=utc)
                choicsModel = Attribute_choices_list

                if dataset == "household":
                    # logger.error("You are Here in Household Section Section '{0}'".format(import_file))
                    model = Household
                    model_A = Household_Attribute
                elif dataset == "personal":
                    # logger.error("You are Here in house Section Section '{0}'".format(import_file))
                    model = Personal
                    model_A = Personal_Attribute
                for key, value in dict(record).items():
                    if value == '':
                        del record[key]
                    elif value != '' and value != 'other':
                        # logger.error( dataQ.count())
                        dataQ = model_A.objects.all().filter(odk_question=key)
                        # logger.error( key+":"+str(dataQ.count()))
                        if dataQ.count() == 1:
                            for r in dataQ:

                                if r.choice_list != '' and not (r.choice_list is None):
                                    # logger.error(r.choice_list)
                                    if r.is_multiple == 'no':
                                        choice_Value = choicsModel.objects.all().filter(list_name=r.choice_list,
                                                                                name=value)
                                        # logger.error(choice_Value.count())
                                        if choice_Value.count() == 1:
                                            for r2 in choice_Value:
                                                record[key] = r2.label
                                                # logger.error(record[key])
                                    elif r.is_multiple == 'yes':
                                        values1 = str(value).split(' ')
                                        valueTxt = []
                                        #    logger.error(values1)
                                        for item in values1:
                                            if item != 'other':
                                                item1 = int(item)
                                                choice_Value = choicsModel.objects.all().filter(list_name=r.choice_list,
                                                                                name=item1)
                                                for r2 in choice_Value:
                                                    valueTxt.append(r2.label)
                                            elif item == 'other':
                                                valueTxt.append(item)
                                                pass
                                        comb = "|".join(valueTxt)

                                        record[key] = comb
                                    #    logger.error(key+":"+record[key])
                        else:
                            logger.error("More or less then one odk_question")

                data = model_A.objects.all().filter(datatype="text")
                for row in data:
                    key = row.odk_question
                    if key in record:
                        record[key] = str(record[key])
                data0 = model_A.objects.all().filter(datatype="date")
                for row in data0:
                    key = row.odk_question
                    # logger.error(key)
                    if key in record:
                        record[key] = datetime.strptime(record[key], '%m/%d/%Y %H:%M')

                c = model.objects.all().filter(KEY=record['KEY'])
                # logger.error(c.count())
                if (c.count() == 0):
                    # logger.error(record['KEY'])
                    m = model(**record)
                    m.save()
                    # logger.error(m)
                    i += 1
                elif (c.count() == 1):
                    # model.objects.filter(KEY=record['KEY']).update(**record)

                    j += 1
                ART = datetime.utcnow().replace(tzinfo=utc)
                ART_diff = (ART - iniciate).total_seconds()
                onerecod = (ART - Rtime).total_seconds()
                logger.error(
                    "TotalTime:{7}, time for this:{8}, {4} dataset: {3} out of {0} data processed of which {1} Data added and {2} data already Exit".format(
                        TR, i, j, i + j, model.__name__, iniciate, ART, ART_diff, onerecod))

            return render(request, 'smartprofile/import.html', {'nbar': "importData", 'importdataform': importdataform,
                                                                'Massage': "Initial Time:{5}, Now:{6}, TimeDiff:{7}, {4} dataset: {3} out of {0} data processed of which {1} Data added and {2} data already Exit".format(
                                                                    TR, i, j, i + j, model.__name__, iniciate, ART,
                                                                    ART_diff)})
        logger.error((datetime.utcnow().replace(tzinfo=utc) - iniciate).total_seconds())

    return render(request, 'upload/import.html', {'nbar': "importData", 'importdataform': importdataform})


import requests
from bs4 import BeautifulSoup
import csv
from multiprocessing import Pool
from re import compile as com

class CarCodesScraper:
    def __init__(self):
        links = []
        for i in ('', '1', '2', '3'):
            x = requests.get("http://www.troublecodes.net/p%scodes/" % i)
            b = BeautifulSoup(x.text)
            links.extend([a['href'] for a in b.find_all('a', {'href': com("http://www.troublecodes.net/p*")})])

        print("Starting pool")
        p = Pool(processes=4)
        with open('troublecodes.csv', 'w') as fp:
            writer = csv.writer(fp)
            for res in p.imap_unordered(self.scrape, links):
                writer.writerow(res)
                print("Succesfully written %s" % res[0])

    @staticmethod
    def scrape(link: str) -> list:
        response = requests.get(link).text
        ret = [link.split('/')[-2], BeautifulSoup(response).select("td:nth-of-type(2)")[0].text]

        def find(text):
            temp = response.find(text)
            if temp != -1:
                code = response[temp:]
                code = code[:code[4:].find("
")+4]
                ret.append(' '.join([tag.text for tag in BeautifulSoup(code).find_all('p')]))
            else:
                ret.append('')

        find("
What Does ")
        find("
Symptoms of ")
        find("
Common Causes of ")
        find("
Troubleshooting ")
        ret.append(link)
        return ret

CarCodesScraper()


import os
import shutil
import tempfile

from tartangan import inception_utils
from tartangan.trainers.components.base import TrainerComponent
from tartangan.utils.cli import type_or_none

class FIDComponent(TrainerComponent):
    """Calculate Frechet Inception Distance"""

    def on_train_begin(self, steps, logs):
        if self.trainer.args.cleanup_inception_model:
            self.model_path = tempfile.mkdtemp()
            os.environ['TORCH_HOME'] = self.model_path
            print(f'Setting $TORCH_HOME to {self.model_path}')

        print('Preparing inception metrics...')
        self.get_inception_metrics = inception_utils.prepare_inception_metrics(
            self.trainer.args.inception_moments, self.trainer.device, False
        )

    def on_train_end(self, steps, logs):
        if self.trainer.args.cleanup_inception_model:
            print(f'Cleaning up $TORCH_HOME = {self.model_path}')
            shutil.rmtree(self.model_path)

    def on_batch_end(self, steps, logs):
        """Calculate inception metrics"""
        if steps and steps % self.trainer.args.fid_freq == 0:
            print('Calculating inception metrics...')
            is_mean, is_std, fid = self._calculate()
            logs['fid'].append(fid)
            logs['inception_score_mean'].append(is_mean)
            logs['inception_score_std'].append(is_std)

    def _calculate(self):
        is_mean, is_std, fid = self.get_inception_metrics(
            self.trainer.sample_g, self.trainer.args.n_inception_imgs,
            num_splits=5
        )
        print('Inception Score is %3.3f +/- %3.3f' % (is_mean, is_std))
        print('FID is %5.4f' % (fid,))
        return is_mean, is_std, fid

    @classmethod
    def add_args_to_parser(self, parser):
        parser.add_argument('--inception-moments', type=type_or_none(str), default=None,
                            help='Path to pre-calculated inception moments')
        parser.add_argument('--n-inception-imgs', default=1000, type=int)
        parser.add_argument('--cleanup-inception-model', action='store_true',
                            help='Delete pretrained inception model used for FID metric.')
        parser.add_argument('--fid-freq', default=10000, type=int,
                            help='Calculate test metrics every N batches')


#!/usr/bin/env python3
import requests
from bs4 import BeautifulSoup
import pdb
import pandas as pd
import time
import csv

def download_raw_data(page=0):
    """Download data."""
    link = f"https://www.gunviolencearchive.org/last-72-hours?page={page}"
    headers = {
        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36"
    }

    data = requests.get(link, headers=headers)
    if data.status_code != 200:
        return None
    return data.text

def extract_table_data(html):
    '''Extract data table from html text.'''
    soup = BeautifulSoup(html, "html.parser")
    table = soup.find("table")
    table_data = []
    for row in table.find_all("tr"):
        items = row.find_all("td")
        # Skip header row
        if not items:
            continue
        row_data = [item.text.strip() for item in items]
        row_data[-1] = items[-1].find_all("a")[-1]['href']
        table_data.append(row_data)
    return table_data

def download_last72hours():
    '''Download all data within the last 72 hours.'''
    data = []
    for pageid in range(50):
        raw_html = download_raw_data(pageid)
        table = extract_table_data(raw_html)
        data.extend(table)
        if len(table) < 25:
            break
    return data

if __name__ == '__main__':
    data = download_last72hours()
    date = time.strftime('%Y-%m-%d')
    outfile = f'data/data-{date}.csv'
    with open(outfile, 'w') as f:
        writer = csv.writer(f)
        writer.writerow(['id', 'date', 'state', 'county',
                         'addr', 'death', 'injured', 'link'])
        writer.writerows(data)
        print(f'{len(data)} rows written to {outfile}')


import random
from urllib.request import urlopen
import sys

##URL
WORD_URL = "http://learncodethehardway.org/words.txt"
WORDS=[]

##Map having Key value pair
PHRASES = {
    "class %%%(%%%):":"Make a class named %%% that is-a %%%",
    "class %%%(object):\n\tdef __init__(self,***)":"class %%% has-a init that takes self and *** params.",
    "class %%%(object):\n\tdef ***(self,@@@)":"class %%% has-a function *** that taked self and @@@ params.",
    "*** = %%%()":"Set *** to a instance of class %%%.",
    "***.***(@@@)":"From *** get the *** function,and call it with param self and @@@.",
    "***.***  = \'***\'":"From *** get the *** attribute and set it to \'***\'."
}

#do you want to drill phrases first
if len(sys.argv) == 2 and sys.argv[1] == "english":
    PHRASES_FIRST = True
else:
    PHRASES_FIRST = False

##Load up the word from website
for word in urlopen(WORD_URL).readlines():
    #print(word)
    WORDS.append(str(word.strip(),encoding = 'utf-8'))

def convert(snippet,phrase):
    class_names = [w.capitalize() for w in random.sample(WORDS,snippet.count("%%%"))]
    #print("class_names: ",class_names)
    other_names = random.sample(WORDS,snippet.count("***"))
    #print("other_names: ",other_names)

    results = []
    param_names = []

    for i in range(0,snippet.count("@@@")):
        param_count = random.randint(1,3)
        param_names.append(', '.join(random.sample(WORDS,param_count)))

    for sentence in snippet,phrase:
        print(type(sentence))
        result = sentence[:]
        print(result)
        for word in class_names:
            result = result.replace("%%%",word,1)

        for word in other_names:
            result = result.replace("***",word,1)

        for word in param_names:
            result = result.replace("@@@",word,1)

        results.append(result)

    return results

try:
    while True:
        snippets = list(PHRASES.keys())
        random.shuffle(snippets)

        for snippet in snippets:
            phrase = PHRASES[snippet]
#            #question,answer = c
            question,answer = convert(snippet,phrase)
#            convert(snippet,phrase)
            if PHRASES_FIRST:
               question,answer = answer,question

            print(question)
            input("> ")
            print(f"ANSWER: {answer}\n\n")
except EOFError:
    Print('\nBYE#')


import pickle

file = open("ProductList.txt", "wb")

my_dict = {"CRMS": ['Str ICream',35.0,1.0],
           "CRMC": ['Choco ICream',45.0,1.0],
           "CRMV": ['Vnll ICream',25.0,1.0],
           "PCHP": ['Potato Chips',57.0,1.0],
           "AT":['Gold Wheat Atta',62.0,0.5],
           "BRC":['Basamti Rice',74.0,1.0],
           "VGO":['Vegetable Oil',56.0,0.5],
           "TMT":['Tomato',40.0,1.0],
           "ONI":['Onion',85.0,1.0],
           "MLK":['Cow Milk',14.0,0.5]}

pickle.dump(my_dict, file)

file.close()


import pickle
from datetime import date
RECORDSIZE=50

class CarRecord :
   def __init__(self) :
       VehicleID = ""
       VehicleID = VehicleID.ljust(20)
       self.VehicleID = VehicleID.encode('utf-8')
       Registration = " "
       Registration = Registration.ljust(10)
       self.Registration = Registration.encode('utf-8')
       self.DateOfRegistration = None
       self.EngineSize = 0
       self.PurchasePrice = 0.00

def InitialiseFile() :
    CarFile = open('CarFile.DAT','wb')
    for i in range(100):
      Address = i * RECORDSIZE + 1
      CarFile.seek(Address, 0)
      pickle.dump(CarRecord(), CarFile)
    CarFile.close()

def InputNewRecordData() :
    Car = CarRecord()
    VehicleID = input('Vehicle ID: ')
    VehicleID = VehicleID.ljust(20)
    Car.VehicleID = VehicleID.encode('utf-8')
    Registration = input('Registration: ')
    Registration = Registration.ljust(10)
    Car.Registration = Registration.encode('utf-8')
    Car.DateOfregistration =(input('Registration Date: '));
    Car.EngineSize = int(input('Engine size: '))
    Car.PurchasePrice = float(input('Purchaseprice: '))
    return Car

def Hash(reg) :
    address = ord(reg[1]) * RECORDSIZE + 1
    return address

def SaveToFile() :
    Address = Hash(Car.Registration.decode('utf-8'))
    CarFile.seek(Address, 0)
    pickle.dump(Car, CarFile)

def OpenFileForUpdate() :
    CarFile = open('CarFile.DAT','rb+')
    return CarFile

def FindRecord(reg, CarFile) :
    Address = Hash(reg)
    CarFile.seek(Address, 0)
    Car = pickle.load(CarFile)
    return Car

def OutputData(Car) :
    print("The Vehicle ID is",Car.VehicleID)

InitialiseFile()
CarFile = OpenFileForUpdate()
Car = CarRecord()
Answer = input('add record or not： ')
while Answer != 'no' :
   Car = CarRecord()
   Car = InputNewRecordData()
   SaveToFile()
   Answer = input('add another record or not：  ')
Answer = input('find record or not： ')
while Answer != 'no' :
   Reg = input('enter vehicle registration number: ')
   Car = FindRecord(Reg, CarFile)
   OutputData(Car)
   Answer = input('find another record or not：')
CarFile.close()


import os

from setuptools import setup, find_packages

def read(fname):
    return open(os.path.join(os.path.dirname(__file__), fname)).read()

setup(
    name="Digitacion",
    version="0.1.0",
    url='',
    license='GPL V. 3',
    description="Digitacion de planillas",
    long_description=read('README.md'),

    author='Herald Olivares',
    author_email='heraldmatias.oz@gmail.com',

    packages=find_packages('src'),
    package_dir={'': 'src'},

    install_requires=['setuptools', 'django'],

    classifiers=[
        'Development Status :: Production',
        'Framework :: Django',
        'Intended Audience :: Public',
        'License :: OSI Approved :: GPL License',
        'Operating System :: OS Independent',
        'Programming Language :: Python',
        'Topic :: Software Development :: Statics',
        'Topic :: Internet :: WWW/HTTP',
    ],

    include_package_data=True,
    zip_safe=False,
)


from cgi import test
from ctypes.wintypes import tagRECT
# from msilib import sequence
from operator import mod
from pickletools import optimize
import tensorflow as tf
import numpy as np
import pandas as pd
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Input, Lambda, Dense, Flatten,Conv2D, BatchNormalization, Dropout
from tensorflow.keras.layers import MaxPooling2D
from tensorflow.keras.optimizers import Adam

from glob import glob
import matplotlib.pyplot as plt

# from tensorflow.keras.optimizers import adam
from tensorflow.keras.preprocessing.image import ImageDataGenerator

#Rescaling the images
TrainingData = ImageDataGenerator(rescale = 1./255)
TestingData = ImageDataGenerator(rescale = 1./255)

#Preprocess all train data
train_generator = TrainingData.flow_from_directory(
    'Data/train',
    target_size = (48, 48),
    batch_size = 64,
    color_mode = "grayscale",
    class_mode = "categorical"
)

#Preprocess all test data
test_generator = TestingData.flow_from_directory(
    'Data/test',
    target_size = (48, 48),
    batch_size = 64,
    color_mode = "grayscale",
    class_mode = "categorical"
)

model = Sequential()
model.add(Conv2D(32, (3, 3), padding='same', activation='relu', input_shape=(48, 48, 1)))
model.add(BatchNormalization())
model.add(Conv2D(32, (3, 3), activation='relu'))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2)))

model.add(Conv2D(64, (3, 3), padding='same', activation='relu'))
model.add(BatchNormalization())
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2)))

model.add(Conv2D(128, (3, 3), padding='same', activation='relu'))
model.add(BatchNormalization())
model.add(Conv2D(128, (3, 3), activation='relu'))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2)))

model.add(Flatten())
model.add(Dense(512, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(7, activation='softmax'))

model.compile(
  loss='categorical_crossentropy',
  optimizer='adam',
  metrics=['accuracy']
)

#Training the model
model_info = model.fit_generator(
    train_generator,
    steps_per_epoch = 28709//64,
    epochs = 20,
    validation_data = test_generator,
    validation_steps = 7178//64
)

#Saving our model in json file
model_json = model.to_json()
with open("model.json", "w") as json_file:
    json_file.write(model_json)

#saving trained model weights in .h5 file
model.save("models/emotion.h5")


from collections import namedtuple

Policy = namedtuple("Policy", ["name", "file", "desc"])

Role = namedtuple("Role", ["name", "file"])

EC2_Instance = namedtuple(
    "EC2_Instance", ["ami", "type", "userdata", "cpu_scaling_file"]
)

Sec_Group = namedtuple("Sec_Group", ["port", "name", "description", "proto", "myip"])

Auto_Scaling_Bundle = namedtuple(
    "Auto_Scaling_Bundle",
    [
        "asg_name",
        "asg_min_srv",
        "asg_max_srv",
        "tg_name",
        "tg_port",
        "tg_proto",
        "asg_cpu_scale_out",
    ],
)

Subnet_Bundle = namedtuple("Subnet_Bundle", ["cidr", "az", "subnet_name"])


import pandas as pd
from pathlib import Path
import logging
import re

class _FileObj:
    def __init__(self, path_obj, dataframe=None, dataframe_name=None,
    colname_chars_replace_underscore="", colname_chars_replace_custom={},
    colname_chars_remove="", sample_data=500, interpret_date_timestamp=False,
    interpret_date_timestamp_errors="raise", **kwargs):
        """
        Create a FileObj instance that has a single attribute, df which is a pandas dataframe
        supports xls, xlsx, csv, tsv files. Only the first worksheet in an Excel workbook
        with multiple sheets will be read.
        param: use any keyword parameters valid in pandas.read_csv()
        parameter: dataframe - default None; a pandas DataFrame object to be profiled
        parameter: dataframe_name - default None; a unique string for the DataFrame, will be used as the
            filename for the dataframe profile ex. DataFrame_Name_profile.xlsx
        parameter: colname_chars_replace_underscore - string of invalid characters to be replaced with an underscore
        parameter: colname_chars_replace_custom - dict of characters and their replacement value
        parameter: colname_chars_remove - string of characters to be removed
        parameter: interpret_date_timestamp - boolean default False, attempt to convert string fields to date or timestamp
        parameter: interpret_date_timestamp_errors - text default "raise", options are "raise", "ignore", "coerce".
            "raise" will raise errors on values that cannot be converted, "ignore" will not raise errors and returns the
            input data, "coerce" will return NaT values when they cannot be converted.
        """
        # Initialize logging
        self.log = logging.getLogger()
        self.log.handlers = []
        self.log.setLevel(logging.INFO)
        log_fmt = logging.Formatter('%(levelname)s: %(asctime)s %(thread)d %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
        screen = logging.StreamHandler()
        screen.setFormatter(log_fmt)
        self.log.addHandler(screen)

        self.df = None
        # df_name must be unique to create unique output filenames
        self.df_name = None

        # update default replace with underscore characters with user-defined characters
        escape_string = r'\/()[]{},.!?:;-^~`' + colname_chars_replace_underscore
        self.colname_chars_replace_underscore = re.escape(escape_string) + r'\s+'
        colname_chars_replace_custom_default = {'#': 'num', '$': 'usd', '%': 'pct', '&': 'and', '+': 'plus',
        '*': 'times', '=': 'equals', '<': 'lt', '>': 'gt', '@': 'at', '|': 'or'}

        # set values for interpret_date_timestamp parameters
        if interpret_date_timestamp not in [True, False]:
            raise Exception(f"'{interpret_date_timestamp}' is not a valid value for interpret_date_timestamp.")
        if interpret_date_timestamp is True and interpret_date_timestamp_errors not in ["raise", "ignore", "coerce"]:
            raise Exception(f"'{interpret_date_timestamp_errors}' is not a valid value for interpret_date_timestamp_errors.")
        if interpret_date_timestamp in [True, False] and interpret_date_timestamp_errors in ["raise", "ignore", "coerce"]:
            self.interpret_date_timestamp = interpret_date_timestamp
            self.interpret_date_timestamp_errors = interpret_date_timestamp_errors

        # update colname_chars_replace_custom with user-defined dict
        colname_chars_replace_custom_default.update(colname_chars_replace_custom)

        # create new colname chars replace dict with properly escaped values as needed
        self.colname_chars_replace_custom = {}
        for key, value in colname_chars_replace_custom_default.items():
            self.colname_chars_replace_custom[re.escape(key)] = value

        # set attribute for characters to be removed
        self.colname_chars_remove = colname_chars_remove

        # set sample data attributes
        if (isinstance(sample_data, int) and sample_data > 0) or sample_data is None:
            self.sample_data = sample_data
        else:
            raise Exception('sample_data must be an integer > 0 or None.')

        # log.info('Creating FileObj')
        if path_obj == 'dataframe':
            if dataframe is None:
                raise Exception('No dataframe was assigned to the "dataframe=" argument.')
            else:
                self.df = dataframe
                if dataframe_name is None:
                    raise Exception('If profiling a dataframe, argument dataframe_name must be a unique string')
                else:
                    self.df_name = dataframe_name
        elif path_obj.is_file():
            if path_obj.suffix in('.xls', '.xlsx'):
                try:
                    self.df = pd.read_excel(path_obj, **kwargs)
                except Exception as error:
                    self.log.exception(error)

            elif path_obj.suffix in ['.csv', '.tsv', '.txt']:
                try:
                    self.df = pd.read_csv(path_obj, **kwargs)
                except Exception as error:
                    self.log.exception(error)
        else:
            raise Exception(f'{path_obj.name} is not a file.  Please use a valid text or excel file')

        if self.df is None:
            self.log.warning(f'{path_obj.name} was not parsed, please check file format and kwargs')
        else:
            # log.info('Created FileObj')
            self.id_cols = []
            self.dim_cols = []
            self.path_obj = path_obj
            self.sample_data_export = self.df.copy().head(self.sample_data)

    def get_columns(self):
        """
        Not currently used as the column list is contained in the more comprehensive Data Types output
        returns: dataframe of columns from file
        """
        self.log.info('Retrieving Columns')
        return pd.DataFrame(self.df.columns, columns=['Columns'])

    def get_data_types(self):
        """
        method that sets values for self.id_cols and self.dim_cols, additionally the
        returns: dataframe of column names and their data types
        """
        self.log.info('Retrieving Data Types')

        if self.interpret_date_timestamp:
            # attempt explicit date transformation for object cols not automatically detected as dates
            self.df = self.df.apply(lambda x: self.convert_to_datetime(x, self.interpret_date_timestamp_errors))

        df = pd.DataFrame(self.df.dtypes, columns=['Data Type'])
        df.index.name = 'Column Name'
        df = df.reset_index()

        df['Clean Column Name'] = self.clean_column_names(df['Column Name'])

        # identify ID columns
        id_col_pat = re.compile(r"(?:^id$|^ID$)|(?:[-_\s]+id|[-_\s]+ID$)|(?:[a-z]+ID$)|(?:[-_\s]+code)",)
        df['Potential ID Column'] = df['Column Name'].apply(lambda x: True if re.search(id_col_pat, x) else None)
        # set FileObj attribute "ID Columns", referenced in dim_cols below
        self.id_cols = df.loc[df['Potential ID Column'] == True, 'Column Name'].tolist()

        # identify the proper data type
        for col in self.df.columns:
            # set string representation of pandas series dtype
            col_dtype = str(self.df[col].dtype)

            if self.df[col].count() > 0:
                if col_dtype in ['object', 'bool'] or 'datetime' in col_dtype:
                    col_values_precision_series = self.df[col].astype('str').str.len()
                    max_precision_value = max(col_values_precision_series)
                    min_precision_value = min(col_values_precision_series)
                elif col_dtype in ['int', 'int64', 'int32']:
                    max_precision_value = self.df[col].max()
                    min_precision_value = self.df[col].min()
                elif col_dtype in ['float', 'float64', 'float32']:
                    col_values_precision_series = self.df[col].astype('str')
                    col_values_precision_df = col_values_precision_series.str.split('.', expand=True)
                    # precision
                    min_precision_value = max(col_values_precision_df[0].str.len() + col_values_precision_df[1].str.len())
                    # scale
                    max_precision_value = max(col_values_precision_df[1].str.len())
                    # check if the Data Type could be int
                    try:
                        if col_values_precision_df[1].fillna('0').astype('int64').sum() == 0:
                            df.loc[df['Column Name'] == col, 'Data Type'] = 'decimal or integer'
                    except ValueError:
                        # pass because this should catch INT overflow that has no negative affect on output
                        pass
                else:
                    min_precision_value = 'dtype not supported'
                    max_precision_value = 'dtype not supported'
            else:
                min_precision_value = 0
                max_precision_value = 0
                df.loc[df['Column Name'] == col, 'Data Type'] = 'N/A'

            df.loc[df['Column Name'] == col, 'Min Length|Value/Precision'] = min_precision_value
            df.loc[df['Column Name'] == col, 'Max Length|Value/Scale'] = max_precision_value

            # check for NULL values
            if sum(self.df[col].isna()) > 0:
                nullable_value = True
            else:
                nullable_value = None

            df.loc[df['Column Name'] == col, 'Nullable'] = nullable_value

            # set FileObj attribute "Dim Columns"
            if self.df[col].dtype == 'object' and col not in self.id_cols:
                self.dim_cols.append(col)

        # replace obscure data type names with clear names
        replace_dict = {'datetime64[ns]': 'date/datetime', 'object':'text', 'int': 'integer',
        'int8': 'integer', 'int32': 'integer', 'int64': 'integer', 'float': 'decimal',
        'float32': 'decimal', 'float64': 'decimal',}
        df['Data Type'] = df['Data Type'].replace(to_replace=replace_dict)

        # identify potential PII columns
        pii_col_pat = re.compile(r'(?:last|full|first|family|given)\s*\S*(?:name|nm)|' \
                                r'\S*\s*(?:address|addr)|(?:e\S*\s*mail)|' \
                                r'(?:tel|tele)\S*(?:phone|no)|(?:phone)', re.IGNORECASE)
        pii_col_name_df = df['Column Name'].apply(lambda x: True if re.search(pii_col_pat, x) else None)
        pii_col_name_cols = df.loc[pii_col_name_df.notna(), 'Column Name'].to_list()

        # look for columns containing PII not already flagged as potential PII columns
        # columns with telephone numbers
        remaining_columns = [col for col in df['Column Name'] if col not in pii_col_name_cols]
        telephone_pat = re.compile(r'\d{10,13}') # re.compile(r'(?:\(*\+*\d+\-*\.*\s*\(*\d+\)*\-*\.*\s*\d+\-*\.*\s*\d+)')
        tel_no_df = (self.df[remaining_columns].replace(to_replace=r'\D', value='', regex=True)
                        .apply(lambda x: x.astype(str).str.contains(telephone_pat, regex=True), axis=0))
        tel_no_cols = tel_no_df.any(axis=0)
        tel_no_cols = list(tel_no_cols[tel_no_cols].index)

        pii_cols = pii_col_name_cols + tel_no_cols

        # columns with email addresses
        remaining_columns = [col for col in df['Column Name'] if col not in pii_cols]
        email_pat = re.compile(r'(?:\S+@\S+\.\S+)')
        email_df = self.df[remaining_columns].apply(lambda x: x.astype(str).str.contains(email_pat, regex=True), axis=0)
        email_cols = email_df.any(axis=0)
        email_cols = list(email_cols[email_cols].index)

        # add email_cols to list of potential pii fields
        pii_cols += email_cols

        # columns with street addresses
        remaining_columns = []
        for col in df['Column Name']:
            if (col not in pii_cols
                    and not df[(df['Column Name'] == col) & (df['Data Type'] == 'text')].empty):
                remaining_columns.append(col)

        street_address_pat = re.compile(r'(?:\d+\s+[a-zA-Z0-9\-]+\s*[a-zA-Z0-9\-]*)')
        street_address_df = (self.df[remaining_columns]
                                        .apply(lambda x: x.astype(str).str.contains(street_address_pat, regex=True), axis=0))
        street_address_cols = street_address_df.any(axis=0)
        street_address_cols = list(street_address_cols[street_address_cols].index)

        # add street_address_cols to list of potential pii fields
        pii_cols += street_address_cols

        df['Potential PII Column'] = df['Column Name'].apply(lambda x: True if x in pii_cols else None)

        # set FileObj attribute "PII Columns", referenced in dim_cols below
        self.pii_cols = df.loc[df['Potential PII Column'] == True, 'Column Name'].tolist()

        return df[['Column Name', 'Clean Column Name', 'Data Type', 'Min Length|Value/Precision',
            'Max Length|Value/Scale', 'Potential ID Column', 'Potential PII Column', 'Nullable']]

    def clean_column_names(self, colname_series):
        """
        Take a pandas series of column names and apply cleansing steps
        Returns: pandas series of cleaned column names
        """
        colname_series_clean = colname_series

        # remove unwanted characters
        if self.colname_chars_remove != "":
            colname_series_clean = colname_series.str.replace(f'[{self.colname_chars_remove}]+', '', regex=True)

        # Replace unacceptable characters in column names with undercores
        colname_series_clean = colname_series_clean.str.replace(f'[{self.colname_chars_replace_underscore}]+', '_', regex=True)
        # replace chars with custom values
        for char, replacement in self.colname_chars_replace_custom.items():
            colname_series_clean = colname_series_clean.str.replace(f'{char}', f'_{replacement}_', regex=True)

        # replace multiple underscores with a single underscore
        colname_series_clean = colname_series_clean.str.replace(r'_+', '_', regex=True)
        # remove leading and trailing underscores
        colname_series_clean = colname_series_clean.str.strip('_')

        # insert underscore for column names that might be IDs and use camel case
        colname_series_clean = colname_series_clean.apply(lambda x: _modify_camel_case_names(x))

        # lower case the clean column name
        colname_series_clean = colname_series_clean.str.lower()

        return colname_series_clean

    def get_text_distinct_values(self):
        """
        returns: dataframe of column names and series of distinct values
        """
        self.log.info('Retrieving Text Value Distribution')
        results_dict = {}
        pandas_numeric_dtype_list = ['int', 'int64', 'int32', 'float', 'float64', 'float32']
        for col in self.df.columns:
            if self.df[col].dtype in pandas_numeric_dtype_list and col not in self.id_cols:
                results_dict[col] = pd.DataFrame(['NA for numeric columns'], columns=[col])
            else:
                df = pd.DataFrame(self.df[col].value_counts())
                df_null = pd.DataFrame({col: len(self.df[self.df[col].isna()])}, index=['NULL'])
                df = pd.concat([df_null, df], sort=False)
                df.index.name = col
                df.rename(index=str, columns={col: f'{col}_counts'}, inplace=True)
                results_dict[col] = df.reset_index()

        distinct_text_values_df = pd.concat(results_dict.values(), axis=1, join='outer', sort=True)

        distinct_text_values_df = replace_xml_illegal_characters(distinct_text_values_df)

        # strip timezone from output because Excel does not support localized TZ info
        # return distinct_text_values_df.apply(lambda x: x.dt.tz_localize(None) if 'datetime' in str(x.dtype) else x)
        return distinct_text_values_df

    def get_numeric_value_distribution(self):
        """
        returns: dataframe of descriptive stats for numeric columns
        """
        self.log.info('Retrieving Numeric Value Distribution')
        df = pd.DataFrame(self.df.describe()).reset_index()
        df.rename(index=str, columns={'index': 'Stat'}, inplace=True)
        return df

    def get_primary_keys(self):
        """
        analyzes dataframe to see the maximum number of non-metric columns that can be part of a primary key
        this is done by checking for columns that when grouped, retain the same number of distinct records as the
        oringinal dataframe
        returns: dataframe of suggested primary key(s) and dataframe of potential ID fields"""
        self.log.info('Looking for Potential Primary Key(s)')

        pk_1 = []
        pk_2 = []
        df_len = 0
        id_cols = self.id_cols.copy()
        dim_cols = self.dim_cols.copy()
        for col in id_cols + dim_cols:
            new_len = len(self.df.groupby(pk_1 + [col], sort=False).count())
            if new_len > df_len:
                df_len = new_len
                pk_1.append(col)
            else:
                pass

        id_cols.reverse()
        dim_cols.reverse()
        df_len = 0
        for col in id_cols + dim_cols:
            if len(self.df.groupby(pk_2 + [col], sort=False).count()) < df_len:
                pk_2.append(col)
            else:
                pass

        return pd.DataFrame({'Column Name': list(set(pk_1 + pk_2))})

    def create_sample(self):
        if self.sample_data is not None:
            # strip timezone from output because Excel does not support localized TZ info
            # return self.df.head(self.sample_data).apply(lambda x: x.dt.tz_localize(None) if 'datetime' in str(x.dtype) else x)
            return self.sample_data_export

    def convert_to_datetime(self, col, interpret_date_timestamp_errors):
        if str(col.dtype) == "object":
            try:
                new_col = pd.to_datetime(col, errors=interpret_date_timestamp_errors, utc=False)
                # if transformation was completely destructive, reset new_col to col
                if new_col.count() == 0:
                    new_col = col
            except Exception as e:
                self.log.error(f"Failed to cast {col.name} to date/datetime - {e}")
                # initialize new_col
                new_col = col
        else:
            new_col = col

        return new_col

def _modify_camel_case_names(x):
    results = re.findall(r'([a-z][A-Z])', x)
    if results:
        for match in results:
            x = x.replace(match, "_".join(list(match)))
    return x

def replace_xml_illegal_characters(df):
    """
    replace characters that throw ILLEGAL_CHARACTER_ERROR in openpyxl
    when writing to XLS(X) formats, found regex pattern in openpyxl source here:
    https://openpyxl.readthedocs.io/en/stable/_modules/openpyxl/cell/cell.html
    """
    ILLEGAL_CHARACTERS_RE = re.compile(r'[\000-\010]|[\013-\014]|[\016-\037]')

    return df.replace(to_replace=ILLEGAL_CHARACTERS_RE, value='~', regex=True)


import os

import pylons.config
import webhelpers

from pylonsex.config.routing import make_map

def load_environment(global_conf={}, app_conf={}):
    map = make_map(global_conf, app_conf)
    # Setup our paths
    root_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    paths = {'root_path': root_path,
             'controllers': os.path.join(root_path, 'controllers'),
             'templates': [os.path.join(root_path, path) for path in \
                           ('components', 'templates')],
             'static_files': os.path.join(root_path, 'public')
             }

    # The following options are passed directly into Myghty, so all configuration options
    # available to the Myghty handler are available for your use here
    myghty = {}
    myghty['log_errors'] = True
    myghty['escapes'] = dict(l=webhelpers.auto_link, s=webhelpers.simple_format)

    # Add your own Myghty config options here, note that all config options will override
    # any Pylons config options

    # Return our loaded config object
    return pylons.config.Config(myghty, map, paths)