#!python3
# Chapter 11 Project - Pull up google maps to search a given address
# USAGE: MapIt.py <address>

import sys, webbrowser

# Check to make sure command line args exist for the address
if len(sys.argv) < 1:
    print("USAGE: MapIt.py <address>")
    sys.exit()

# Build the search URL for google maps
baseURL = "https://www.google.com/maps/place/"
address = "+".join(sys.argv[1:])
searchURL = baseURL + address

webbrowser.open(searchURL)


def DownloadYoutubeVid(link,SAVE_PATH = "D:\Hajer\Scrapper\ytVideos"    ):
    from pytube import YouTube
    try:
        # object creation using YouTube
        # which was imported in the beginning
        yt = YouTube(link)
    except:
        print("Connection Error") #to handle exception
    try:
        did=False
        a=yt.streams.filter(res="1080p")
        if list(a)==[]:
            a=yt.streams.filter(res="720p")
            if list(a)!=[]:
                a.first().download(output_path = SAVE_PATH, filename = yt.title)
                did=True
        else:
            a.first().download(output_path = SAVE_PATH, filename = yt.title)
            did=True
        if not(did)&(list(yt.streams)!=[]):
            yt.streams.first().download(SAVE_PATH, filename = yt.title)
    except:
        print("Some Error!")
    #os.rename(yt.streams.first().default_filename, 'new_filename.ext')

def getlinks(csvpath="D:\Hajer\Scrapper\ml-youtube.csv"):
    import csv
    links=[]
    with open(csvpath, mode='r',encoding="utf8") as csv_file:
        csv_reader = csv.reader(csv_file, delimiter=',')
        #csv_reader = csv.DictReader(csv_file)
        line_count = 0
        for row in csv_reader:
            if line_count == 0:
                #print(f'Column names are {", ".join(row)}')
                line_count += 1
                continue
            #print(f'\t{row[0]} ---- {row[1]} ---- {row[2]}.')
            link="https://www.youtube.com/watch?v="+row[0]
            links.append(link)
            line_count += 1
        print(f'Processed {line_count} lines.')
    return links

DownloadYoutubeVid("https://www.youtube.com/watch?v=Q5s_HCPdD6E")
#links=getlinks(csvpath="D:\Hajer\Scrapper\ml-youtube.csv")
# for link in links:
#     DownloadYoutubeVid(link)


import numpy as np
import pandas as pd
from geopy.distance import geodesic
from dateutil.parser import parse

def trajcheck(file, lat_diff=1., lon_diff=1., search=None, path = '/home/ollie/muali/Data/winter_all/'):
    """This function checks if a given trajectory is passing through a observation station.
    Outputs the matching hour, Obs Station Name, etc.
    Modified to give distance in km between the traj and station as well

    file_= Name of the time from winter_all folder as'tdump_lvl_MM_DD_HH'

    lat_diff= Minimum latitude difference to satisfy, default 1.0

    lon_diff =Minimum latitude difference to satisfy, default 1.0

    search: (str) to be used for searching a pattern from the output

    path: default is winter_all folder """

    file_name = path + file

    df_IGR = pd.read_excel('/home/ollie/muali/python_notebooks/IGR_25N_v2.xlsx')
    #f = open(file_name,'rb+')

    # getting launch date
    #for i, file_ in enumerate(f):
     #   if i ==4: # reads 6th line
      #      date_str = f.read(25)
       #     f.close()
    #l_date = parse(date_str)

    df = pd.read_csv(file_name, skiprows=7, header=None, delim_whitespace=True)
    # a very efficient way to rename columns
    # dict to rename columns
    cols_renames = {2: 'year', 3: 'month', 4: 'day', 5: 'hour', 6: 'minute'}

    # converting year to 4 digits
    df.iloc[:,2] = df.iloc[:,2] + 1900

    # df_subset is used to merge yy, mm, dd, hh values into BT_time column
    df_subset = df.loc[:, list(cols_renames.keys())].rename(columns=cols_renames)
    # df_subset is extracting the time from the original dataframe to be used to make a time index for the final dataframe
    dt_series = pd.to_datetime(df_subset) # converts the df into a time series which we will add as extra column
    dt_series.head()
    # adding a new backtrajectory column
    df['BT_time'] = dt_series # adding the back trajectory time column to df

    # dropping the not required columns
    df.drop([0,1,2,3,4,5,6,7], axis =1, inplace=True)
    df.head()

    col_rename2 = {8: 'back_hr', 9:'Lat', 10: 'Lon', 11:'AGL', 12:'Pressure'}
    df.rename(columns=col_rename2, inplace=True)

    del df_subset
    del dt_series

    count = 0
    # count will be used to tell how many times our condition is met

    string_list =[] # to store the output of print to be used by regex
    for row_IRG in df_IGR.itertuples():
        for row in df.itertuples():
            # lat_diff lon_diff provided by the user
            if ((-lat_diff < (row_IRG.Lat - row.Lat) < lat_diff) & (-lon_diff < (row_IRG.Lon - row.Lon) < lon_diff)):
              #  print(row.back_hr, row.BT_time, row_IRG.ID, row_IRG.Lat, row_IRG.Lon ,row_IRG.Lat - row.Lat, row_IRG.Lon - row.Lon)
#                print(row.back_hr, row.BT_time.strftime('%d-%m-%Y %H:%M'), row_IRG.ID, row_IRG.Lat, row_IRG.Lon , row.Lat - row_IRG.Lat, row.Lon - row_IRG.Lon, geodesic((row.Lat, row.Lon), (row_IRG.Lat, row_IRG.Lon)).km )
                string_list.append((row.back_hr, row.BT_time.strftime('%d-%m-%Y %H:%M'), row_IRG.ID, row_IRG.Lat, row_IRG.Lon , row.Lat - row_IRG.Lat, row.Lon - row_IRG.Lon, geodesic((row.Lat, row.Lon), (row_IRG.Lat, row_IRG.Lon)).km ))
                count = count + 1

    print("The following are the counts:")
    print(count)

    if search is not None:
        for item in string_list:
            if search in item:
                print(item,'\n')

    else:
        for item in string_list:
            print(item,'\n')
# eof


import numpy as np
import utils
import config as c

# Define log class
class Log:
    def __init__(self):
        # Initialize logs
        self.mu = np.zeros((c.n_trials, c.n_steps, c.n_orders, c.n_joints * 3))
        self.mu_dot = np.zeros_like(self.mu)

        self.actions = np.zeros((c.n_trials, c.n_steps, c.n_joints))
        self.actions_dot = np.zeros_like(self.actions)

        self.angles = np.zeros((c.n_trials, c.n_steps, c.n_joints))
        self.est_angles = np.zeros_like(self.angles)

        self.target_pos = np.zeros((c.n_trials, c.n_steps, 2))
        self.est_target_pos = np.zeros_like(self.target_pos)

        self.hand_pos = np.zeros((c.n_trials, c.n_steps, 2))
        self.est_hand_pos = np.zeros_like(self.hand_pos)

        self.mode = np.zeros((c.n_trials, c.n_steps))
        self.int_errors = np.zeros((c.n_trials, c.n_steps, 2, c.n_joints * 3))

    # Track logs for each time step
    def track(self, trial, step, agent, arm, target_pos):
        self.mu[trial, step] = agent.mu
        self.mu_dot[trial, step] = agent.mu_dot

        self.actions[trial, step] = agent.a
        self.actions_dot[trial, step] = agent.a_dot

        limits = np.tile(agent.limits, c.n_joints)
        mu = utils.denormalize(agent.mu[0], limits)
        self.angles[trial, step] = arm.angles
        self.est_angles[trial, step] = mu[:c.n_joints]

        self.target_pos[trial, step] = target_pos
        self.est_target_pos[trial, step] = arm.kinematics(
            mu[c.n_joints: c.n_joints * 2])[-1, :2]

        self.hand_pos[trial, step] = arm.poses[-1, :2]
        self.est_hand_pos[trial, step] = arm.kinematics(
            mu[:c.n_joints])[-1, :2]

        self.mode[trial, step] = agent.mode

        int_errors = agent.E_i * np.repeat(agent.beta, c.n_joints * 3). \
            reshape(2, c.n_joints * 3)
        self.int_errors[trial, step] = int_errors

    # Save log to file
    def save_log(self):
        np.savez_compressed('simulation/log' + c.log_name,
                            mu=self.mu, mu_dot=self.mu_dot,
                            actions=self.actions,
                            actions_dot=self.actions_dot,
                            angles=self.angles,
                            est_angles=self.est_angles,
                            target_pos=self.target_pos,
                            est_target_pos=self.est_target_pos,
                            hand_pos=self.hand_pos,
                            est_hand_pos=self.est_hand_pos,
                            mode=self.mode, int_errors=self.int_errors)


# Exercício Python 72: Crie um programa que tenha uma dupla totalmente preenchida com uma contagem por extenso, de zero até vinte. Seu programa deverá ler um número pelo teclado (entre 0 e 20) e mostrá-lo por extenso.

numerosPorExtenso = ('Zero', 'Um','Dois','Três','Quatro','Cinco','Seis','Sete','Oito','Nove','Dez','Onze','Doze','Treze','Quatorze','Quinze','Dezesseis','Dezessete','Dezoito','Dezenove','Vinte')

for cont in range(0,len(numerosPorExtenso)):
    opNumeros= int(input(' Digite um número entre 0 e 20: '))
    if opNumeros >= 0 and opNumeros <= 20:
        print(f'{numerosPorExtenso[opNumeros]}')
    else:
        print('Tente novamente.', end = '')


import datetime
from unittest.mock import MagicMock
import pytest

from app.dictionary.enums import DictionaryValueType
from app.dictionary.models import DictionaryItem
from app.dictionary.mapper import DictionaryMapper
from app.dictionary.schemas import KVItem, Dictionary

@pytest.fixture
def mapper():
    return DictionaryMapper

@pytest.fixture
def dictionary_objects():
    return [
        MagicMock(
            DictionaryItem=DictionaryItem(
                key='eth_price_usd',
                value='100',
                value_type=DictionaryValueType.INTEGER,
                updated=datetime.datetime(2022, 2, 22, 22, 22, 22),
            )
        ),
        MagicMock(
            DictionaryItem=DictionaryItem(
                key='last_block_timestamp',
                value='1600000042',
                value_type=DictionaryValueType.INTEGER,
                updated=datetime.datetime(2011, 1, 11, 11, 11, 11),
            )
        ),
    ]

def test_map_to_dictionary_items(mapper, dictionary_objects):
    items = mapper.map_to_dictionary_items(dictionary_objects)

    assert all(isinstance(item, KVItem) for item in items)

    assert items == [
        KVItem(
            key='eth_price_usd',
            value='100',
            value_type=DictionaryValueType.INTEGER,
            updated=datetime.datetime(2022, 2, 22, 22, 22, 22).timestamp(),
        ),
        KVItem(
            key='last_block_timestamp',
            value='1600000042',
            value_type=DictionaryValueType.INTEGER,
            updated=datetime.datetime(2011, 1, 11, 11, 11, 11).timestamp(),
        ),
    ]

def test_map_to_dictionary(mapper, dictionary_objects):
    items = mapper.map_to_dictionary_items(dictionary_objects)
    dictionary = mapper.map_to_dictionary(items)

    assert dictionary == Dictionary(eth_price_usd=100, last_block_timestamp=1600000042)

def test_map_to_dictionary_empty(mapper):
    dictionary = mapper.map_to_dictionary([])

    assert dictionary == Dictionary()


import json
from pathlib import Path
from collections import defaultdict

counts = defaultdict(int)

def walk(root: Path):
    for thing in root.iterdir():
        if thing.is_dir():
            walk(thing)
        elif thing.name == "index.json":
            with open(thing) as f:
                doc = json.load(f)["doc"]
                flaws = doc.get("flaws") or {}
                for flawgroup in flaws:
                    if flawgroup == "macros":
                        for macroflaw in flaws[flawgroup]:
                            group = f"macros__{macroflaw['name']}"
                            counts[group] += 1
                    else:
                        counts[flawgroup] += 1

walk(Path("client/build/en-us/docs/"))

# flat = sorted[(v,k) for k,v in counts.items()]
# from pprint import pprint
# pprint(counts)
for flaw in sorted(counts):
    print(flaw.ljust(35), f"{counts[flaw]:,}")


import asyncio
from typing import List

import discord
from discord.ext import commands

from settings import rulebook_url, tourney_name, footer_icon, footer_note, \
    veto_timeout, newline
from smash.game import Game
from smash.player import Player
from utils.checks import playerCheck, stageCheck

class Match:
    """
    Represents a Smash match
    """

    def __init__(self, player1: Player, player2: Player, num_of_games: int):
        # players
        self.player1 = player1
        self.player2 = player2
        self.winner = None

        # match data
        self.games: List[Game] = []
        self.name: str = f"{tourney_name}: {player1.name} vs {player2.name}"
        self.description: str = f"{self.player1.mention} vs " \
                                f"{self.player2.mention} " \
                                f"\nThe rulebook can be found " \
                                f"[here]({rulebook_url})"

        # match state
        self.num_of_games: int = num_of_games
        self.current_game: int = 0

        # generate first game
        self.games.append(Game(0))

    @property
    def embed(self):
        title = f"Smash Ultimate Best-of-{self.num_of_games} Veto"
        embed = discord.Embed(title=title,
                              description=self.description,
                              color=discord.Colour.gold())
        # loop through max games times and generate embed fields
        for x in range(len(self.games)):
            embed.add_field(name=self.games[x].name,
                            value=self.games[x].winner_embed(),
                            inline=False)
            # x - 1 because its using index num
            embed.add_field(name="__Starter Stages__",
                            value=self.games[x].starters_embed(),
                            inline=True)
            embed.add_field(name="__Counterpick Stages__",
                            value=self.games[x].counters_embed(),
                            inline=True)
        # set footer
        embed.set_footer(icon_url=footer_icon,
                         text=f"{tourney_name} | {footer_note}")

        return embed

    async def veto(self, ctx: discord.ext.commands.Context,
                   bot: discord.ext.commands.Bot):
        # FIGURING OUT GAME EMBED INDEXES (ONLY IF U WANT TO REWRITE):
        # GAME NUM * 3 = embed title index, just + 1 and + 2 for stage lists
        # IF FINISH EARLY LOOP FROM CURRENT_GAME TO NUM_OF_GAMES - 1
        # AND CROSS OUT LINE

        # add DSR stages to the game if on game 3 or higher
        if self.current_game >= 2:
            for x in range(len(self.player2.dsr)):
                self.games[self.current_game].veto_stage(
                    self.player2.dsr[x].name)

        # send first embed
        await ctx.send(embed=self.embed)

        # initial veto
        if self.current_game == 0:
            # loop through 4 stage veto's/selection
            for x in range(4):
                # check which message to send
                if x == 0:
                    await ctx.send(f"{self.player1.mention}, veto a starter.")
                if x == 1:
                    await ctx.send(f"{self.player2.mention}, veto a starter.")
                if x == 2:
                    await ctx.send(
                        f"{self.player2.mention}, veto another starter.")
                if x == 3:
                    await ctx.send(
                        f"{self.player1.mention}, select the stage from "
                        f"the remaining starters.")

                # wait for players stage choice
                msg = await bot.wait_for('message',
                                         check=stageCheck(ctx, self),
                                         timeout=veto_timeout)

                # if on stage selection
                if x == 3:
                    # choose the stage
                    self.games[self.current_game].choose_stage(msg.content)
                # if on veto still
                else:
                    # veto the stage
                    self.games[self.current_game].veto_stage(msg.content)

                # regenerate and send embed
                await ctx.send(embed=self.embed)

        # subsequent veto
        else:
            # loop through 3 stage veto's/selection process
            for x in range(3):
                if x == 0:
                    await ctx.send(
                        f"{self.player1.mention}, veto a stage.")
                elif x == 1:
                    await ctx.send(
                        f"{self.player1.mention}, veto another stage.")
                elif x == 2:
                    await ctx.send(
                        f"{self.player2.mention}, select the stage.")

                # wait for players stage choice
                msg = await bot.wait_for('message',
                                         check=stageCheck(ctx, self),
                                         timeout=veto_timeout)

                # if on stage selection
                if x == 2:
                    # choose the stage
                    self.games[self.current_game].choose_stage(msg.content)
                # if on veto still
                else:
                    # veto the stage
                    self.games[self.current_game].veto_stage(msg.content)

                # regenerate and send embed
                await ctx.send(embed=self.embed)

        # get winner
        await ctx.send(f"{newline}GLHF! Once finished, "
                       f"the winner should say `me`.")
        msg = await bot.wait_for('message',
                                 check=playerCheck(ctx),
                                 timeout=veto_timeout)

        # swap player 1 if player 2 was winner
        if msg.author.id == self.player2.id:
            self.swap_players()

        # add stage to winner dsr list
        self.player1.dsr.append(
            self.games[self.current_game].selected_stage)

        # set the game winner
        self.games[self.current_game].winner = self.player1
        self.player1.wins += 1

        # check if the match has a winner
        if self.player1.wins >= (self.num_of_games // 2) + 1:
            self.winner = self.player1
            await ctx.send(embed=self.embed)
            await ctx.send('GG! Run `ve!close` to archive this channel.')

        # else prep for the next game veto
        else:
            # move to next game
            self.current_game += 1

            # add games
            self.games.append(Game(self.current_game))

            # setup next game
            await ctx.send(embed=self.embed)
            await ctx.send('Starting next game veto...')
            await asyncio.sleep(3)

    def swap_players(self):
        """
        Swap player1 and player2
        """
        # swap player objects
        p1 = self.player2
        p2 = self.player1

        # set new players
        self.player1 = p1
        self.player2 = p2


import sys
import pandas as pd

not_included_gene_file = './not_included_genes.txt'
gene_file = '../../ecoli_refgene/ecoli_refgene.txt'
raw_file = './raw_data.txt'

# load data
with open(not_included_gene_file) as f:
    not_included_genes = f.readlines()
not_included_genes = [row.strip() for row in not_included_genes]

pd_refgenes = pd.read_csv(gene_file, sep='\t')
genes = []
for name, name2 in zip(pd_refgenes['name'], pd_refgenes['name2']):
    if name not in not_included_genes or name2 not in not_included_genes:
        genes.append(name2)

antibiotics = []
knowledge = []
triples = []
first = True

pd_raw_data = pd.read_csv(raw_file, sep='\t')
for row in pd_raw_data.itertuples(index=False):
    if row[2] not in antibiotics:
        antibiotics.append(row[2])
    triples.append(list(row))

for antibiotic in antibiotics:
    for gene in genes:
        pos_triple = [gene, 'upregulated by antibiotic after 30 mins', antibiotic]
        neg_triple = [gene, 'not upregulated by antibiotic after 30 mins', antibiotic]
        if pos_triple not in triples:
            triples.append(neg_triple)

pd_data = pd.DataFrame(triples, columns=['Subject', 'Predicate', 'Object'])
pd_data = pd_data.drop_duplicates()
pd_data.to_csv('./Shaw_et_al_intermediate.txt', sep='\t', index=False)


# -*- coding: utf-8 -*-
import connection as mqtt
from time import sleep
from wa_calc import *
import datetime
import time
import sqlite3

mqtt.connect()

mqtt.publish_retained("display/2fc4f5fc/config","4,0,1,2,3")

# conn=sqlite3.connect("log.db")
# c=conn.cursor()

# c.execute("""CREATE TABLE IF NOT EXISTS logbook (
#                                         latitude real,
#                                         longtitude real,
#                                         tws real,
#                                         twa integer,
#                                         aws real,
#                                         awa integer,
#                                         bs real,
#                                         heel real,
#                                         dpth real,
#                                         heading integer,
#                                         sec_time integer
#                                     )""")

counter_mins = 5
starttime = int(time.time() + counter_mins*60)
mqtt.publish("race/starttime",starttime)

print("**Simulation Started**")
boat_gps=gps_coords(40, 55, 'N', 'E')
while True:

    aw = vec(realistic_aws(),random.randint(3355,3755)/100)
    aawm = average_aw_mag(aw.mag)
    #aw.mag=aawm
    bs = boat_speed(aawm)
    COG = vec(bs, 180)
    heading = realistic_heading(120)#set the heading
    tw = vec_add(aw,COG)
    boat_gps=generate_gps(heading, bs, boat_gps)
    sleep(0.3)

    tws  = round(tw.mag,1)
    twa  = round(boat_to_compass(heading,tw.angle),1)
    awa  = round(abs(aw.angle),1)
    aws  = round(aw.mag,1)
    bsr  = round(bs,1)
    dpth = realistic_depth()
    heel = realistic_heel(aw)

    twawithzero = str(int(tw.angle))
    if len(twawithzero) < 3 :
        twawithzero = "0" + twawithzero
    awawithzero = str(int(awa))
    if len(awawithzero) < 3 :
        awawithzero = "0" + awawithzero

    # bundle = str(tws) +","+ str(round(tw.angle,1)) +","+ str(aws) +","+ str(awa) +","+ str(bsr) +","+ str(dpth) +","+ str(heel)
    # print (bundle)
    # c.execute("INSERT INTO logbook VALUES (?,?,?,?,?,?,?,?,?,?,?)",(boat_gps.latitude,boat_gps.longtitude,tws,twa,aws,awa,bsr,heel,dpth,heading,int(time.time())))
    # conn.commit()
    # mqtt.publish("COORDINATES latitude", boat_gps.latitude)
    # mqtt.publish("COORDINATES longtitude", boat_gps.longtitude)

    mqtt.publish("wind/tws",tws)
    mqtt.publish("wind/twa",twawithzero)
    mqtt.publish("wind/aws",str(aws))
    mqtt.publish("wind/awa",awawithzero)
    mqtt.publish("boat/speed",str(bsr))
    mqtt.publish("boat/heel" ,str(heel))
    mqtt.publish("boat/depth",str(dpth))
    mqtt.publish("boat/heading",str(heading))

    # mqtt.publish("bundle/all",bundle)

# conn.close()


import sys
import time
import pyshark
import socket
import pickle
import random
import hashlib
import argparse
import ipaddress
from sklearn.feature_extraction.text import CountVectorizer
from multiprocessing import Process, Manager, Value, Queue
from util_functions import *

# Sample commands
# split a pcap file into smaller chunks to leverage multi-core CPUs: tcpdump -r dataset.pcap -w dataset-chunk -C 1000
# dataset parsing (first step): python3 lucid_dataset_parser.py --dataset_type SYN2020 --dataset_folder ./sample-dataset/ --packets_per_flow 10 --dataset_id SYN2020 --traffic_type all --time_window 10
# dataset parsing (second step): python3 lucid_dataset_parser.py --preprocess_folder ./sample-dataset/

IDS2018_DDOS_FLOWS = {'attackers': ['18.218.115.60', '18.219.9.1','18.219.32.43','18.218.55.126','52.14.136.135','18.219.5.43','18.216.200.189','18.218.229.235','18.218.11.51','18.216.24.42'],
                      'victims': ['18.218.83.150','172.31.69.28']}

IDS2017_DDOS_FLOWS = {'attackers': ['172.16.0.1'],
                      'victims': ['192.168.10.50']}

CUSTOM_DDOS_SYN = {'attackers': ['11.0.0.' + str(x) for x in range(1,255)],
                      'victims': ['10.42.0.2']}

DOS2019_FLOWS = {'attackers': ['172.16.0.5'], 'victims': ['192.168.50.1', '192.168.50.4']}

DDOS_ATTACK_SPECS = {
    'DOS2017' : IDS2017_DDOS_FLOWS,
    'DOS2018' : IDS2018_DDOS_FLOWS,
    'SYN2020' : CUSTOM_DDOS_SYN,
    'DOS2019': DOS2019_FLOWS
}

vector_proto = CountVectorizer()
vector_proto.fit_transform(protocols).todense()

random.seed(SEED)
np.random.seed(SEED)

class packet_features:
    def __init__(self):
        self.id_fwd = (0,0,0,0,0) # 5-tuple src_ip_addr, src_port,,dst_ip_addr,dst_port,protocol
        self.id_bwd = (0,0,0,0,0)  # 5-tuple src_ip_addr, src_port,,dst_ip_addr,dst_port,protocol
        self.features_list = []

    def __str__(self):
        return "{} -> {}".format(self.id_fwd, self.features_list)

def get_ddos_flows(attackers,victims):
    DDOS_FLOWS = {}

    if '/' in attackers: # subnet
        DDOS_FLOWS['attackers'] = [str(ip) for ip in list(ipaddress.IPv4Network(attackers).hosts())]
    else: # single address
        DDOS_FLOWS['attackers'] = [str(ipaddress.IPv4Address(attackers))]

    if '/' in victims:  # subnet
        DDOS_FLOWS['victims'] = [str(ip) for ip in list(ipaddress.IPv4Network(victims).hosts())]
    else:  # single address
        DDOS_FLOWS['victims'] = [str(ipaddress.IPv4Address(victims))]

    return DDOS_FLOWS

# function that build the labels based on the dataset type
def parse_labels(dataset_type=None, attackers=None,victims=None, label=1):
    output_dict = {}

    if attackers is not None and victims is not None:
        DDOS_FLOWS = get_ddos_flows(attackers, victims)
    elif dataset_type is not None and dataset_type in DDOS_ATTACK_SPECS:
        DDOS_FLOWS = DDOS_ATTACK_SPECS[dataset_type]
    else:
        return None

    for attacker in DDOS_FLOWS['attackers']:
        for victim in DDOS_FLOWS['victims']:
            ip_src = str(attacker)
            ip_dst = str(victim)
            key_fwd = (ip_src, ip_dst)
            key_bwd = (ip_dst, ip_src)

            if key_fwd not in output_dict:
                output_dict[key_fwd] = label
            if key_bwd not in output_dict:
                output_dict[key_bwd] = label

    return output_dict

def parse_packet(pkt):
    pf = packet_features()
    tmp_id = [0,0,0,0,0]

    try:
        pf.features_list.append(float(pkt.sniff_timestamp))  # timestampchild.find('Tag').text
        pf.features_list.append(int(pkt.ip.len))  # packet length
        pf.features_list.append(int(hashlib.sha256(str(pkt.highest_layer).encode('utf-8')).hexdigest(),
                                    16) % 10 ** 8)  # highest layer in the packet
        pf.features_list.append(int(int(pkt.ip.flags, 16)))  # IP flags
        tmp_id[0] = str(pkt.ip.src)  # int(ipaddress.IPv4Address(pkt.ip.src))
        tmp_id[2] = str(pkt.ip.dst)  # int(ipaddress.IPv4Address(pkt.ip.dst))

        protocols = vector_proto.transform([pkt.frame_info.protocols]).toarray().tolist()[0]
        protocols = [1 if i >= 1 else 0 for i in
                     protocols]  # we do not want the protocols counted more than once (sometimes they are listed twice in pkt.frame_info.protocols)
        protocols_value = int(np.dot(np.array(protocols), powers_of_two))
        pf.features_list.append(protocols_value)

        protocol = int(pkt.ip.proto)
        tmp_id[4] = protocol
        if pkt.transport_layer != None:
            if protocol == socket.IPPROTO_TCP:
                tmp_id[1] = int(pkt.tcp.srcport)
                tmp_id[3] = int(pkt.tcp.dstport)
                pf.features_list.append(int(pkt.tcp.len))  # TCP length
                pf.features_list.append(int(pkt.tcp.ack))  # TCP ack
                pf.features_list.append(int(pkt.tcp.flags, 16))  # TCP flags
                pf.features_list.append(int(pkt.tcp.window_size_value))  # TCP window size
                pf.features_list = pf.features_list + [0, 0]  # UDP + ICMP positions
            elif protocol == socket.IPPROTO_UDP:
                pf.features_list = pf.features_list + [0, 0, 0, 0]  # TCP positions
                tmp_id[1] = int(pkt.udp.srcport)
                pf.features_list.append(int(pkt.udp.length))  # UDP length
                tmp_id[3] = int(pkt.udp.dstport)
                pf.features_list = pf.features_list + [0]  # ICMP position
        elif protocol == socket.IPPROTO_ICMP:
            pf.features_list = pf.features_list + [0, 0, 0, 0, 0]  # TCP and UDP positions
            pf.features_list.append(int(pkt.icmp.type))  # ICMP type
        else:
            pf.features_list = pf.features_list + [0, 0, 0, 0, 0, 0]  # padding for layer3-only packets
            tmp_id[4] = 0

        pf.id_fwd = (tmp_id[0], tmp_id[1], tmp_id[2], tmp_id[3], tmp_id[4])
        pf.id_bwd = (tmp_id[2], tmp_id[3], tmp_id[0], tmp_id[1], tmp_id[4])

        return pf

    except AttributeError as e:
        # ignore packets that aren't TCP/UDP or IPv4
        return None

# Offline preprocessing of pcap files for model training, validation and testing
def process_pcap(pcap_file,dataset_type,in_labels,max_flow_len,labelled_flows,max_flows=0, traffic_type='all',time_window=TIME_WINDOW):
    start_time = time.time()
    temp_dict = OrderedDict()
    start_time_window = -1

    pcap_name = pcap_file.split("/")[-1]
    print("Processing file: ", pcap_name)

    cap = pyshark.FileCapture(pcap_file)
    for i, pkt in enumerate(cap):
        if i % 1000 == 0:
            print(pcap_name + " packet #", i)

        # start_time_window is used to group packets/flows captured in a time-window
        if start_time_window == -1 or float(pkt.sniff_timestamp) > start_time_window + time_window:
            start_time_window = float(pkt.sniff_timestamp)

        pf = parse_packet(pkt)
        store_packet(pf, temp_dict, start_time_window, max_flow_len)
        if max_flows > 0 and len(temp_dict) >= max_flows:
            break

    apply_labels(temp_dict, labelled_flows, in_labels, traffic_type)
    print('Completed file {} in {} seconds.'.format(pcap_name, time.time() - start_time))

# Transforms live traffic into input samples for inference
def process_live_traffic(cap, dataset_type, in_labels, max_flow_len, traffic_type='all',time_window=TIME_WINDOW):
    start_time = time.time()
    temp_dict = OrderedDict()
    labelled_flows = []

    start_time_window = start_time
    time_window = start_time_window + time_window

    if isinstance(cap, pyshark.LiveCapture) == True:
        for pkt in cap.sniff_continuously():
            if time.time() >= time_window:
                break
            pf = parse_packet(pkt)
            temp_dict = store_packet(pf, temp_dict, start_time_window, max_flow_len)
    elif isinstance(cap, pyshark.FileCapture) == True:
        while time.time() < time_window:
            try:
                pkt = cap.next()
                pf = parse_packet(pkt)
                temp_dict = store_packet(pf,temp_dict,start_time_window,max_flow_len)
            except:
                break

    apply_labels(temp_dict,labelled_flows, in_labels,traffic_type)
    return labelled_flows

def store_packet(pf,temp_dict,start_time_window, max_flow_len):
    if pf is not None:
        if pf.id_fwd in temp_dict and start_time_window in temp_dict[pf.id_fwd] and \
                temp_dict[pf.id_fwd][start_time_window].shape[0] < max_flow_len:
            temp_dict[pf.id_fwd][start_time_window] = np.vstack(
                [temp_dict[pf.id_fwd][start_time_window], pf.features_list])
        elif pf.id_bwd in temp_dict and start_time_window in temp_dict[pf.id_bwd] and \
                temp_dict[pf.id_bwd][start_time_window].shape[0] < max_flow_len:
            temp_dict[pf.id_bwd][start_time_window] = np.vstack(
                [temp_dict[pf.id_bwd][start_time_window], pf.features_list])
        else:
            if pf.id_fwd not in temp_dict and pf.id_bwd not in temp_dict:
                temp_dict[pf.id_fwd] = {start_time_window: np.array([pf.features_list]), 'label': 0}
            elif pf.id_fwd in temp_dict and start_time_window not in temp_dict[pf.id_fwd]:
                temp_dict[pf.id_fwd][start_time_window] = np.array([pf.features_list])
            elif pf.id_bwd in temp_dict and start_time_window not in temp_dict[pf.id_bwd]:
                temp_dict[pf.id_bwd][start_time_window] = np.array([pf.features_list])
    return temp_dict

def apply_labels(flows, labelled_flows, labels, traffic_type):
    for five_tuple, flow in flows.items():
        if labels is not None:
            short_key = (five_tuple[0], five_tuple[2])  # for IDS2017/IDS2018 dataset the labels have shorter keys
            flow['label'] = labels.get(short_key, 0)

        for flow_key, packet_list in flow.items():
            # relative time wrt the time of the first packet in the flow
            if flow_key != 'label':
                amin = np.amin(packet_list,axis=0)[0]
                packet_list[:, 0] = packet_list[:, 0] - amin

        if traffic_type == 'ddos' and flow['label'] == 0: # we only want malicious flows from this dataset
            continue
        elif traffic_type == 'benign' and flow['label'] > 0: # we only want benign flows from this dataset
            continue
        else:
            labelled_flows.append((five_tuple,flow))

# returns the total number of flows
def count_flows(preprocessed_flows):
    ddos_flows = 0
    total_flows = len(preprocessed_flows)
    ddos_fragments = 0
    total_fragments = 0
    for flow in preprocessed_flows:
        flow_fragments = len(flow[1]) - 1
        total_fragments += flow_fragments
        if flow[1]['label'] > 0:
            ddos_flows += 1
            ddos_fragments += flow_fragments  # the label does not count

    return (total_flows, ddos_flows, total_flows - ddos_flows), (total_fragments, ddos_fragments, total_fragments-ddos_fragments)

# balance the dataset based on the number of benign and malicious fragments of flows
def balance_dataset(flows,total_fragments=float('inf')):
    new_flow_list = []

    _,(_, ddos_fragments, benign_fragments) = count_flows(flows)

    if ddos_fragments == 0 or benign_fragments == 0:
        min_fragments = total_fragments
    else:
        min_fragments = min(total_fragments/2,ddos_fragments,benign_fragments)

    random.shuffle(flows)
    new_benign_fragments = 0
    new_ddos_fragments = 0

    for flow in flows:
        if flow[1]['label'] == 0 and (new_benign_fragments < min_fragments ):
            new_benign_fragments += len(flow[1]) - 1
            new_flow_list.append(flow)
        elif flow[1]['label'] > 0 and (new_ddos_fragments < min_fragments):
            new_ddos_fragments += len(flow[1]) - 1
            new_flow_list.append(flow)

    return new_flow_list, new_benign_fragments, new_ddos_fragments

# convert the dataset from dictionaries with 5-tuples keys into a list of flow fragments and another list of labels
def dataset_to_list_of_fragments(dataset):
    keys = []
    X = []
    y = []

    for flow in dataset:
        tuple = flow[0]
        flow_data = flow[1]
        label = flow_data['label']
        for key, fragment in flow_data.items():
            if key != 'label':
                X.append(fragment)
                y.append(label)
                keys.append(tuple)

    return X,y,keys

def train_test_split(flow_list,train_size=TRAIN_SIZE, shuffle=True):
    test_list = []
    _,(total_examples,_,_) = count_flows(flow_list)
    test_examples = total_examples - total_examples*train_size

    if shuffle == True:
        random.shuffle(flow_list)

    current_test_examples = 0
    while current_test_examples < test_examples:
        flow = flow_list.pop(0)
        test_list.append(flow)
        current_test_examples += len(flow[1])-1

    return flow_list,test_list

def main(argv):
    command_options = " ".join(str(x) for x in argv[1:])

    help_string = 'Usage[0]: python3 lucid_dataset_parser.py --dataset_type <dataset_name> --dataset_folder <folder path> --dataset_id <dataset identifier> --packets_per_flow <n> --time_window <t>\n' \
                  'Usage[1]: python3 lucid_dataset_parser.py --preprocess_folder <folder path>'
    manager = Manager()

    parser = argparse.ArgumentParser(
        description='Dataset parser',
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    parser.add_argument('-d', '--dataset_folder', nargs='+', type=str,
                        help='Folder with the dataset')
    parser.add_argument('-o', '--output_folder', nargs='+', type=str,
                        help='Output folder')
    parser.add_argument('-f', '--traffic_type', default='all', nargs='+', type=str,
                        help='Type of flow to process (all, benign, ddos)')
    parser.add_argument('-p', '--preprocess_folder', nargs='+', type=str,
                        help='Folder with preprocessed data')
    parser.add_argument('--preprocess_file', nargs='+', type=str,
                        help='File with preprocessed data')
    parser.add_argument('-b', '--balance_folder', nargs='+', type=str,
                        help='Folder where balancing datasets')
    parser.add_argument('-n', '--packets_per_flow', nargs='+', type=str,
                        help='Packet per flow sample')
    parser.add_argument('-s', '--samples', default=float('inf'), type=int,
                        help='Number of training samples in the reduced output')
    parser.add_argument('-i', '--dataset_id', nargs='+', type=str,
                        help='String to append to the names of output files')
    parser.add_argument('-m', '--max_flows', default=0, type=int,
                        help='Max number of flows to extract from the pcap files')
    parser.add_argument('-l', '--label', default=1, type=int,
                        help='Label assigned to the DDoS class')

    parser.add_argument('-t', '--dataset_type', nargs='+', type=str,
                        help='Type of the dataset. Available options are: DOS2017, DOS2018, DOS2019, SYN2020')

    parser.add_argument('-w', '--time_window', nargs='+', type=str,
                        help='Length of the time window')

    parser.add_argument('--no_split', help='Do not split the dataset', action='store_true')

    args = parser.parse_args()

    if args.packets_per_flow is not None:
        max_flow_len = int(args.packets_per_flow[0])
    else:
        max_flow_len = MAX_FLOW_LEN

    if args.time_window is not None:
        time_window = float(args.time_window[0])
    else:
        time_window = TIME_WINDOW

    if args.dataset_id is not None:
        dataset_id = str(args.dataset_id[0])
    else:
        dataset_id = ''

    if args.traffic_type is not None:
        traffic_type = str(args.traffic_type[0])
    else:
        traffic_type = 'all'

    if args.dataset_folder is not None and args.dataset_type is not None:
        process_list = []
        flows_list = []

        if args.output_folder is not None and os.path.isdir(args.output_folder[0]) is True:
            output_folder = args.output_folder[0]
        else:
            output_folder = args.dataset_folder[0]

        filelist = glob.glob(args.dataset_folder[0]+ '/*.pcap')
        in_labels = parse_labels(args.dataset_type[0],args.dataset_folder[0],label=args.label)

        start_time = time.time()
        for file in filelist:
            try:
                flows = manager.list()
                p = Process(target=process_pcap,args=(file,args.dataset_type[0],in_labels,max_flow_len,flows,args.max_flows, traffic_type,time_window))
                process_list.append(p)
                flows_list.append(flows)
            except FileNotFoundError as e:
                continue

        for p in process_list:
            p.start()

        for p in process_list:
            p.join()

        np.seterr(divide='ignore', invalid='ignore')
        try:
            preprocessed_flows = list(flows_list[0])
        except:
            print ("ERROR: No traffic flows. \nPlease check that the dataset folder name (" + args.dataset_folder[0] + ") is correct and \nthe folder contains the traffic traces in pcap format (the pcap extension is mandatory)")
            exit(1)

        #concatenation of the features
        for results in flows_list[1:]:
            preprocessed_flows = preprocessed_flows + list(results)

        process_time = time.time()-start_time

        if dataset_id == '':
            dataset_id = str(args.dataset_type[0])

        filename = str(int(time_window)) + 't-' + str(max_flow_len) + 'n-' + dataset_id + '-preprocess'
        output_file = output_folder + '/' + filename
        output_file = output_file.replace("//", "/") # remove double slashes when needed

        with open(output_file + '.data', 'wb') as filehandle:
            # store the data as binary data stream
            pickle.dump(preprocessed_flows, filehandle)

        (total_flows, ddos_flows, benign_flows),  (total_fragments, ddos_fragments, benign_fragments) = count_flows(preprocessed_flows)

        log_string = time.strftime("%Y-%m-%d %H:%M:%S") + " | dataset_type:" + args.dataset_type[0] + \
                     " | flows (tot,ben,ddos):(" + str(total_flows) + "," + str(benign_flows) + "," + str(ddos_flows) + \
                     ") | fragments (tot,ben,ddos):(" + str(total_fragments) + "," + str(benign_fragments) + "," + str(ddos_fragments) + \
                     ") | options:" + command_options + " | process_time:" + str(process_time) + " |\n"
        print (log_string)

        # saving log file
        with open(output_folder + '/history.log', "a") as myfile:
            myfile.write(log_string)

    if args.preprocess_folder is not None or args.preprocess_file is not None:
        if args.preprocess_folder is not None:
            output_folder = args.output_folder[0] if args.output_folder is not None else args.preprocess_folder[0]
            filelist = glob.glob(args.preprocess_folder[0] + '/*.data')
        else:
            output_folder = args.output_folder[0] if args.output_folder is not None else os.path.dirname(os.path.realpath(args.preprocess_file[0]))
            filelist = args.preprocess_file

        # obtain time_window and flow_len from filename and ensure that all files have the same values
        time_window = None
        max_flow_len = None
        dataset_id = None
        for file in filelist:
            filename = file.split('/')[-1].strip()
            current_time_window = int(filename.split('-')[0].strip().replace('t',''))
            current_max_flow_len = int(filename.split('-')[1].strip().replace('n',''))
            current_dataset_id = str(filename.split('-')[2].strip())
            if time_window != None and current_time_window != time_window:
                print ("Incosistent time windows!!")
                exit()
            else:
                time_window = current_time_window
            if max_flow_len != None and current_max_flow_len != max_flow_len:
                print ("Incosistent flow lengths!!")
                exit()
            else:
                max_flow_len = current_max_flow_len

            if dataset_id != None and current_dataset_id != dataset_id:
                dataset_id = "IDS201X"
            else:
                dataset_id = current_dataset_id

        preprocessed_flows = []
        for file in filelist:
            with open(file, 'rb') as filehandle:
                # read the data as binary data stream
                preprocessed_flows = preprocessed_flows + pickle.load(filehandle)

        # balance samples and redux the number of samples when requested
        preprocessed_flows, benign_fragments, ddos_fragments = balance_dataset(preprocessed_flows,args.samples)

        if len(preprocessed_flows) == 0:
            print("Empty dataset!")
            exit()

        preprocessed_train, preprocessed_test = train_test_split(preprocessed_flows,train_size=TRAIN_SIZE, shuffle=True)
        preprocessed_train, preprocessed_val = train_test_split(preprocessed_train, train_size=TRAIN_SIZE, shuffle=True)

        X_train, y_train, _ = dataset_to_list_of_fragments(preprocessed_train)
        X_val, y_val, _ = dataset_to_list_of_fragments(preprocessed_val)
        X_test, y_test, _ = dataset_to_list_of_fragments(preprocessed_test)

        # normalization and padding
        X_full = X_train + X_val + X_test
        y_full = y_train + y_val + y_test
        mins,maxs = static_min_max(time_window=time_window)

        total_examples = len(y_full)
        total_ddos_examples = np.count_nonzero(y_full)
        total_benign_examples = total_examples - total_ddos_examples

        output_file = output_folder + '/' + str(time_window) + 't-' + str(max_flow_len) + 'n-' + dataset_id + '-dataset'
        if args.no_split == True: # don't split the dataset
            norm_X_full = normalize_and_padding(X_full, mins, maxs, max_flow_len)
            #norm_X_full = padding(X_full,max_flow_len) # only padding
            norm_X_full_np = np.array(norm_X_full)
            y_full_np = np.array(y_full)

            hf = h5py.File(output_file + '-full.hdf5', 'w')
            hf.create_dataset('set_x', data=norm_X_full_np)
            hf.create_dataset('set_y', data=y_full_np)
            hf.close()

            [full_packets] = count_packets_in_dataset([norm_X_full_np])
            log_string = time.strftime("%Y-%m-%d %H:%M:%S") + " | Total examples (tot,ben,ddos):(" + str(total_examples) + "," + str(total_benign_examples) + "," + str(total_ddos_examples) + \
                         ") | Total packets:(" + str(full_packets) + \
                         ") | options:" + command_options + " |\n"
        else:
            norm_X_train = normalize_and_padding(X_train,mins,maxs,max_flow_len)
            norm_X_val = normalize_and_padding(X_val, mins, maxs, max_flow_len)
            norm_X_test = normalize_and_padding(X_test, mins, maxs, max_flow_len)

            norm_X_train_np = np.array(norm_X_train)
            y_train_np = np.array(y_train)
            norm_X_val_np = np.array(norm_X_val)
            y_val_np = np.array(y_val)
            norm_X_test_np = np.array(norm_X_test)
            y_test_np = np.array(y_test)

            hf = h5py.File(output_file + '-train.hdf5', 'w')
            hf.create_dataset('set_x', data=norm_X_train_np)
            hf.create_dataset('set_y', data=y_train_np)
            hf.close()

            hf = h5py.File(output_file + '-val.hdf5', 'w')
            hf.create_dataset('set_x', data=norm_X_val_np)
            hf.create_dataset('set_y', data=y_val_np)
            hf.close()

            hf = h5py.File(output_file + '-test.hdf5', 'w')
            hf.create_dataset('set_x', data=norm_X_test_np)
            hf.create_dataset('set_y', data=y_test_np)
            hf.close()

            [train_packets, val_packets, test_packets] = count_packets_in_dataset([norm_X_train_np, norm_X_val_np, norm_X_test_np])
            log_string = time.strftime("%Y-%m-%d %H:%M:%S") + " | examples (tot,ben,ddos):(" + str(total_examples) + "," + str(total_benign_examples) + "," + str(total_ddos_examples) + \
                         ") | Train/Val/Test sizes: (" + str(norm_X_train_np.shape[0]) + "," + str(norm_X_val_np.shape[0]) + "," + str(norm_X_test_np.shape[0]) + \
                         ") | Packets (train,val,test):(" + str(train_packets) + "," + str(val_packets) + "," + str(test_packets) + \
                         ") | options:" + command_options + " |\n"

        print(log_string)

        # saving log file
        with open(output_folder + '/history.log', "a") as myfile:
            myfile.write(log_string)

    if args.balance_folder is not None and args.output_folder is not None:
        output_folder = args.output_folder[0] if args.output_folder is not None else args.balance_folder[0]
        datasets = []
        for folder in args.balance_folder:
            datasets += glob.glob(folder + '/*.hdf5')
        train_filelist = {}
        val_filelist = {}
        test_filelist = {}
        min_samples_train = float('inf')
        min_samples_val = float('inf')
        min_samples_test = float('inf')

        output_filename_prefix = None

        for file in datasets:
            filename = file.split('/')[-1].strip()
            dataset = h5py.File(file, "r")
            X = np.array(dataset["set_x"][:])  # features
            Y = np.array(dataset["set_y"][:])  # labels
            if 'train' in filename:
                key = filename.split('dataset')[0].strip() + 'dataset-balanced-train.hdf5'
                if output_filename_prefix ==None:
                    output_filename_prefix = filename.split('IDS')[0].strip()
                else:
                    if filename.split('IDS')[0].strip() != output_filename_prefix:
                        print ("Inconsistent datasets!")
                        exit()
                train_filelist[key] = (X,Y)
                if X.shape[0] < min_samples_train:
                    min_samples_train = X.shape[0]
            elif 'val' in filename:
                key = filename.split('dataset')[0].strip() + 'dataset-balanced-val.hdf5'
                if output_filename_prefix ==None:
                    output_filename_prefix = filename.split('IDS')[0].strip()
                else:
                    if filename.split('IDS')[0].strip() != output_filename_prefix:
                        print ("Inconsistent datasets!")
                        exit()
                val_filelist[key] = (X,Y)
                if X.shape[0] < min_samples_val:
                    min_samples_val = X.shape[0]
            elif 'test' in filename:
                key = filename.split('dataset')[0].strip() + 'dataset-balanced-test.hdf5'
                if output_filename_prefix ==None:
                    output_filename_prefix = filename.split('IDS')[0].strip()
                else:
                    if filename.split('IDS')[0].strip() != output_filename_prefix:
                        print ("Inconsistent datasets!")
                        exit()
                test_filelist[key] = (X, Y)
                if X.shape[0] < min_samples_test:
                    min_samples_test = X.shape[0]

        final_X = {'train':None,'val':None,'test':None}
        final_y = {'train':None,'val':None,'test':None}

        for key,value in train_filelist.items():
            X_short = value[0][:min_samples_train,...]
            y_short = value[1][:min_samples_train,...]

            if final_X['train'] is None:
                final_X['train'] = X_short
                final_y['train'] = y_short
            else:
                final_X['train'] = np.vstack((final_X['train'],X_short))
                final_y['train'] = np.hstack((final_y['train'],y_short))

        for key,value in val_filelist.items():
            X_short = value[0][:min_samples_val,...]
            y_short = value[1][:min_samples_val,...]

            if final_X['val'] is None:
                final_X['val'] = X_short
                final_y['val'] = y_short
            else:
                final_X['val'] = np.vstack((final_X['val'],X_short))
                final_y['val'] = np.hstack((final_y['val'],y_short))

        for key,value in test_filelist.items():
            X_short = value[0][:min_samples_test,...]
            y_short = value[1][:min_samples_test,...]

            if final_X['test'] is None:
                final_X['test'] = X_short
                final_y['test'] = y_short
            else:
                final_X['test'] = np.vstack((final_X['test'],X_short))
                final_y['test'] = np.hstack((final_y['test'],y_short))

        for key,value in final_X.items():
            filename = output_filename_prefix + 'IDS201X-dataset-balanced-' + key + '.hdf5'
            hf = h5py.File(output_folder + '/' + filename, 'w')
            hf.create_dataset('set_x', data=value)
            hf.create_dataset('set_y', data=final_y[key])
            hf.close()

        total_flows = final_y['train'].shape[0]+final_y['val'].shape[0]+final_y['test'].shape[0]
        ddos_flows = np.count_nonzero(final_y['train'])+np.count_nonzero(final_y['val'])+np.count_nonzero(final_y['test'])
        benign_flows = total_flows-ddos_flows
        [train_packets, val_packets, test_packets] = count_packets_in_dataset([final_X['train'], final_X['val'], final_X['test']])
        log_string = time.strftime("%Y-%m-%d %H:%M:%S") + " | total_flows (tot,ben,ddos):(" + str(total_flows) + "," + str(benign_flows) + "," + str(ddos_flows) + \
                     ") | Packets (train,val,test):(" + str(train_packets) + "," + str(val_packets) + "," + str(test_packets) + \
                     ") | Train/Val/Test sizes: (" + str(final_y['train'].shape[0]) + "," + str(final_y['val'].shape[0]) + "," + str(final_y['test'].shape[0]) + \
                     ") | options:" + command_options + " |\n"

        print(log_string)

        # saving log file
        with open(output_folder + '/history.log', "a") as myfile:
            myfile.write(log_string)

    if args.dataset_folder is None and args.preprocess_folder is None and args.preprocess_file is None and args.balance_folder is None:
        print (help_string)
    if args.dataset_type is None and args.dataset_folder is not None:
        print("Please specify the dataset type (DOS2017, DOS2018, DOS2020)!")
        print(help_string)
    if args.output_folder is None and args.balance_folder is not None:
        print("Please specify the output folder!")
        print(help_string)

if __name__ == "__main__":
    main(sys.argv)


import Lanovenko.validator.lib as Lval
import Rodina.validator.lab as Rval

def dataset_common_insert(dataset):
    host_id = input("Host ID: ")
    while not Lval.host_id_validator(host_id):
        print("Expression is invalid! Try again")
        host_id = input("Host ID: ")
    else:
        host_id = int(host_id)

    beds = input("Beds: ")
    while not Lval.beds_validator(beds):
        print("Expression is invalid! Try again")
        beds = input("Beds: ")
    else:
        beds = int(beds)

    number_of_reviews = input("Number of reviews: ")
    while not Lval.number_of_reviews_validator(number_of_reviews):
        print("Expression is invalid! Try again")
        number_reviews = input("Numbers of reviews: ")
    else:
        host_id = int(number_of_reviews)

    country = input("Country: ")
    while not Lval.country_validator(country):
        print("Expression is invalid! Try again")
        country = input("Country: ")

    guests_included = input("Guests included: ")
    while not Rval.guests_included_validator(guests_included):
        print("Expression is invalid! Try again")
        guests_included = input("Guests included: ")
    else:
        guests_included = int(guests_included)

    room_type = input("Room type: ")
    while not Rval.room_type_validator(room_type):
        print("Expression is invalid! Try again")
        room_type = input("Room type: ")

    amenities = input("Amenities: ")
    while not Rval.amenities_validator(amenities):
        print("Expression is invalid! Try again")
        amenities = input("Amenities: ")

    dataset.update(host_id,
                   dict([('country', country),
                         ('beds', beds),
                         ('number_of reviews', number_of_reviews),
                         ('guests_included', guests_included),
                         ('room_type', room_type),
                         ('amenities', amenities)]))


"""
You are climbing a stair case. It takes n steps to reach to the top.

Each time you can either climb 1 or 2 steps. In how many distinct ways can you climb to the top?
"""
import math
def ClimbStaNaive(n):
    return ClimbStaNaive_Aux(0, n)
#End def

def ClimbStaNaive_Aux(i, j):
    if i > j:
        return 0
    else:
        if i == j :
            return 1
        return ClimbStaNaive_Aux(i + 1, j) + ClimbStaNaive_Aux(i + 2, j)
    #End if
#End def

#Memoizado
def ClimbStaMem(n):
    M = [math.inf for x in range(n + 2)]
    return ClimbStaNaive_Aux(0, n)
#End def
def ClimbStaMem_Aux(i, j):
    if M[i] == math.inf:
        if i > j:
            M[i] = 0
        else:
            if i == j :
                M[i] = 1
            else:
                M[i] = ClimbStaNaive_Aux(i + 1, j) + ClimbStaNaive_Aux(i + 2, j)
            #End if
        #End if
    #End if
    return M[i]
#End def

#Bottom-Up
def ClimbStaBottom(n):
    M = [0 for x in range(n)]
    M[0] = 1
    M[1] = 2
    for i in range(2, n):
        M[i] = M[i - 1] + M[i - 2]
    #End for
    return M[n - 1]
#End def

#End def

print(ClimbStaNaive(4))
print(ClimbStaMem(4))
print(ClimbStaBottom(4))


# On 1 TitanX, it takes around 22 hours for training
# metircs (pre, rec, pairwise fscore)
# test on 2 proposal params (gt iop, 0.2-0.8): (99.1, 67.29, 80.16)
# test on 20 proposal params (gt iop, 0.2-0.8): (97.83, 81.2, 88.74)

import os.path as osp
from functools import partial
from proposals import generate_proposals

use_random_seed = True
featureless = False

# model
model = dict(type='dsgcn',
             kwargs=dict(feature_dim=256,
                         featureless=featureless,
                         stage='seg',
                         reduce_method='no_pool',
                         use_random_seed=use_random_seed,
                         hidden_dims=[512, 64]))

# training args
optimizer = dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=1e-4)
optimizer_config = {}

lr_config = dict(
    policy='step',
    step=[15, 24, 28],
)

iter_size = 32
batch_size_per_gpu = 1
test_batch_size_per_gpu = 256
total_epochs = 30
workflow = [('train', 1)]

# misc
workers_per_gpu = 1

checkpoint_config = dict(interval=1)

log_level = 'INFO'
log_config = dict(interval=100, hooks=[
    dict(type='TextLoggerHook'),
])

# post_process
th_pos = -1
th_iou = 1
th_outlier = 0.5
keep_outlier = True

# testing metrics
metrics = ['pairwise', 'bcubed', 'nmi']

# data locations
prefix = './data'
train_name = 'part0_train'
test_name = 'part1_test'

th_iop_min=0.2
th_iop_max=0.8

knn_method = 'hnsw'

k_lst_i0 = [30, 60, 80]
step_i0 = 0.05
minsz_i0 = 3
maxsz_i0 = 300
th_lst_i0 = [0.6, 0.65, 0.7, 0.75]

k_th_lst_i0 = []
for k in k_lst_i0:
    for th in th_lst_i0:
        k_th_lst_i0.append((k, th))

th_i1 = 0.4
step_i1 = 0.05
minsz_i1 = 3
maxsz_i1 = 500
sv_minsz_i1 = 2
k_maxsz_lst_i1 = [(2, 8), (2, 12), (2, 16), (3, 5), (3, 10), (4, 4)]

proposal_params = [
    dict(k=k_i0,
         knn_method=knn_method,
         th_knn=th_i0,
         th_step=step_i0,
         minsz=minsz_i0,
         maxsz=maxsz_i0,
         iter1_params=[
             dict(k=k_i1,
                  knn_method=knn_method,
                  th_knn=th_i1,
                  th_step=step_i1,
                  minsz=minsz_i1,
                  maxsz=maxsz_i1,
                  sv_minsz=sv_minsz_i1,
                  sv_maxsz=sv_maxsz_i1) for k_i1, sv_maxsz_i1 in k_maxsz_lst_i1
         ]) for k_i0, th_i0 in k_th_lst_i0
]
feat_path = osp.join(prefix, 'features', '{}.bin'.format(train_name))
label_path = osp.join(prefix, 'labels', '{}.meta'.format(train_name))
proposal_path = osp.join(prefix, 'cluster_proposals')
train_data = dict(wo_weight=False,
                  feat_path=feat_path,
                  label_path=label_path,
                  use_random_seed=use_random_seed,
                  featureless=featureless,
                  th_iop_min=th_iop_min,
                  th_iop_max=th_iop_max,
                  proposal_folders=partial(generate_proposals,
                                           params=proposal_params,
                                           prefix=prefix,
                                           oprefix=proposal_path,
                                           name=train_name,
                                           dim=model['kwargs']['feature_dim'],
                                           no_normalize=False))

step_i0 = 0.05
minsz_i0 = 3
maxsz_i0 = 300
th_iter0_lst_i0 = [(0.6, True), (0.7, True), (0.75, False)]

th_i1 = 0.4
step_i1 = 0.05
minsz_i1 = 3
maxsz_i1 = 500
sv_minsz_i1 = 2
k_maxsz_lst_i1 = [(2, 8), (2, 12), (2, 16), (3, 5), (3, 10), (4, 4)]

proposal_params = [
    dict(k=k,
         knn_method=knn_method,
         th_knn=th_i0,
         th_step=step_i0,
         minsz=minsz_i0,
         maxsz=maxsz_i0,
         iter0=iter0,
         iter1_params=[
             dict(k=k_i1,
                  knn_method=knn_method,
                  th_knn=th_i1,
                  th_step=step_i1,
                  minsz=minsz_i1,
                  maxsz=maxsz_i1,
                  sv_minsz=sv_minsz_i1,
                  sv_maxsz=sv_maxsz_i1) for k_i1, sv_maxsz_i1 in k_maxsz_lst_i1
         ]) for th_i0, iter0 in th_iter0_lst_i0
]
# use the same proposal params as training
feat_path = osp.join(prefix, 'features', '{}.bin'.format(test_name))
label_path = osp.join(prefix, 'labels', '{}.meta'.format(test_name))
test_data = dict(wo_weight=True,
                 feat_path=feat_path,
                 label_path=label_path,
                 use_random_seed=use_random_seed,
                 use_max_degree_seed=True,
                 featureless=featureless,
                 th_iop_min=th_iop_min,
                 th_iop_max=th_iop_max,
                 proposal_folders=partial(generate_proposals,
                                          params=proposal_params,
                                          prefix=prefix,
                                          oprefix=proposal_path,
                                          name=test_name,
                                          dim=model['kwargs']['feature_dim'],
                                          no_normalize=False))


__author__ = 'George-Zhangzhiping'

'''
brackets pairs check with stack structure
'''

def brackets(pairstr):
    prechs = '{[('
    sufchs = ')]}'
    stack = list()
    for ch in pairstr:
        #print('str: {}'.format(ch))
        if ch in prechs:
            stack.append(ch)
        else:
            popch = stack.pop()
            if ch == ')' and popch != '(' \
                or ch == ']' and popch != '[' \
                or ch == '}' and popch != '{':
                return False
    if len(stack) == 0:
        return  True
    else:
        return  False

pairstr1 = '{[()]}{}[]()'
pairstr2 = '{[}()[]}{}[]()'

assert brackets(pairstr1) is True
print('===')
assert brackets(pairstr2) is False


import cv2
import numpy as np
import mediapipe as mp
import matplotlib.pyplot as plt
import math
import os
import dlib
import pandas as pd
from localize import Localize
from cnnlstm import CNNLSTM
from evaluation import MyEvaluation
from typing import List, Mapping, Optional, Tuple, Union

df = pd.read_csv("CAS(ME)^2evaluation.csv")

test_video = [
            r"D:\CASME\CASME(2)\rawvideo\rawvideo\s15\15_0102disgustingteeth.avi"
            #   r"D:\CASME\CASME(2)\rawvideo\rawvideo\s36\36_0401girlcrashing.avi",
            #   r"D:\CASME\CASME(2)\rawvideo\rawvideo\s36\36_0505funnyinnovations.avi",

            #   r"D:\CASME\CASME(2)\rawvideo\rawvideo\s37\37_0101disgustingteeth.avi",
            #   r"D:\CASME\CASME(2)\rawvideo\rawvideo\s37\37_0402beatingpregnantwoman.avi",
            #   r"D:\CASME\CASME(2)\rawvideo\rawvideo\s37\37_0502funnyerrors.avi",
            #   r"D:\CASME\CASME(2)\rawvideo\rawvideo\s37\37_0505funnyinnovations.avi",
            #   r"D:\CASME\CASME(2)\rawvideo\rawvideo\s37\37_0507climbingthewall.avi",
            #   r"D:\CASME\CASME(2)\rawvideo\rawvideo\s37\37_0508funnydunkey.avi",

            #   r"D:\CASME\CASME(2)\rawvideo\rawvideo\s38\38_0502funnyerrors.avi",
            #   r"D:\CASME\CASME(2)\rawvideo\rawvideo\s38\38_0507climbingthewall.avi",

            #   r"D:\CASME\CASME(2)\rawvideo\rawvideo\s40\40_0401girlcrashing.avi",
            #   r"D:\CASME\CASME(2)\rawvideo\rawvideo\s40\40_0502funnyerrors.avi",
            #   r"D:\CASME\CASME(2)\rawvideo\rawvideo\s40\40_0503unnyfarting.avi"
            ]

localization_algorithm = Localize()
# Initialize Spotting Algorithm
prediction_model = CNNLSTM()
# self.prediction_model.mobilenet("core/transfer_mobilenet_cnnlstm_tfrecord_2/cp.ckpt")
prediction_model.mobilenet("D:\CodeProject2\SKRIPSI_FINAL\core\Weights\Transfer_mobilenet_cnnlstm_localize_tfrecord_pyramid_1/cp.ckpt")

# Pyramid
# "D:\CodeProject2\SKRIPSI_FINAL\core\Transfer_mobilenet_cnnlstm_localize_tfrecord_pyramid_1/cp.ckpt"
# "D:\CodeProject2\SKRIPSI_FINAL\core\Transfer_mobilenet_localize_cnnlstm_tfrecord/cp.ckpt"

eval = MyEvaluation()

for video in test_video:
    # find the detection
    raw_filename = video.split('\\')[-1][:-4]
    find_in_df = df[(df['raw_path'] == raw_filename)]

    detection = []
    for index, item in find_in_df.iterrows():
        onset = int(item['onset'])
        apex = int(item['apex'])
        offset = int(item['offset'])

        if offset == 0:
            offset = apex

        detection.append((onset, offset))

    print("Filename:", raw_filename)
    print("Detection:", detection)
    # Evaluation Loop
    cap = cv2.VideoCapture(video)

    arm_the_localization = True
    detected = True
    xleft = 0
    ytop = 0
    xright = 0
    ybot = 0

    # Read the file
    sliding_window = []
    itr = 0
    while True:
        ret, frame = cap.read()
        if not ret:
            print("Can't receive frame (stream end?). Exiting ...")
            # Process
            break

        itr += 1
        # Our operations on the frame come here
        rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
        if arm_the_localization:
            arm_the_localization = False
            detected, xleft, ytop, xright, ybot = localization_algorithm.mp_face_mesh_crop_preprocessing(frame)

        cropped = frame[ytop:ybot, xleft:xright]
        # Display the resulting frame
        cv2.imshow('frame', cropped)

        # Process the frame using CNN-LSTM
        frame = cv2.resize(cropped, (224, 224), interpolation=cv2.INTER_AREA)
        rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) / 255

        sliding_window.append(rgb)
        if len(sliding_window) > 12:
            sliding_window.pop(0)

        if len(sliding_window) == 12:
            np_sliding_window = np.expand_dims(np.array(sliding_window), axis=0)
            conf, label = prediction_model.process(np_sliding_window, conf=0.7)
            start_frame = itr - 11
            end_frame = itr

            sliding_window = []

            print("Label of frame", start_frame, "to", end_frame, "Label", label, "Confidence Level:", conf)

            eval.count_casme2(start_frame, end_frame, label, detection)

        if cv2.waitKey(1) == ord('q'):
            break
    # When everything done, release the capture
    cap.release()
    cv2.destroyAllWindows()

    eval.to_csv(raw_filename)
    eval.print_total()
    eval.reset_count()
    print("==============================")

print("Evaluation Done")


# para ter acesso as bibliotecas primeiramente instale-as através do terminal, segue comando de instalação:
# pip install pdfminer3k
from pdfminer.pdfinterp import PDFResourceManager, process_pdf
from pdfminer.converter import TextConverter
from pdfminer.layout import LAParams
from io import StringIO
from urllib.request import urlopen

def ler_pdf(arquivopdf):
    # PDFResourceManager Usado para armazenar recursos compartilhados
    # como fontes e imagens
    recursos = PDFResourceManager()
    buffer = StringIO()
    layout_params = LAParams()
    dispositivo = TextConverter(recursos, buffer, laparams=layout_params)
    process_pdf(recursos, dispositivo, arquivopdf)
    dispositivo.close()
    conteudo = buffer.getvalue()
    buffer.close()
    return conteudo

# Abrindo arquivo PDF online
pdf_lido = 'http://imagens.seplag.ce.gov.br/PDF/20210719/do20210719p01.pdf'
arquivoPDF = urlopen(pdf_lido)
stringSaida = ler_pdf(arquivoPDF)
# print(stringSaida)
arquivoPDF.close()
leitura = stringSaida.split('*** *** ***')
# print(leitura)
print(len(leitura))
lista = []
flag_acerto = 0
for i in range(len(leitura)):
    if 'Licitação' in leitura[i] or 'LICITAÇÃO' in leitura[i]:
        if flag_acerto == 0:
            texto = leitura[i].split('AVISO DE')
            lista.append('AVISO DE' + texto[-1])
        # elif i < len(leitura) - 1:
        else:
            lista.append(leitura[i])
        flag_acerto += 1
        # else:
        #     texto = leitura[i].split('*** *** ***')
        #     lista.append(texto[0])
print(len(lista))
print(f'Foram encontrados {flag_acerto} casos.')
arquivo = open('saida.txt', 'w', encoding='UTF-8')
arquivo.write('Dados retirados do pdf encontrado em: ' + pdf_lido + '\n')
arquivo.write('-' * 150)
arquivo.write('\n')
for i in lista:
    # print(i)
    # print('-' * 150)
    arquivo.write(i)
    arquivo.write('-' * 120)
arquivo.close()
# print(stringSaida)


import tempfile

import pkg_resources
from flask import g
from invenio_files_rest.views import ObjectResource

try:
    pkg_resources.get_distribution('wand')
    from wand.image import Image
    HAS_IMAGEMAGICK = True
except pkg_resources.DistributionNotFound:
    # Python module not installed
    HAS_IMAGEMAGICK = False
except ImportError:
    # ImageMagick notinstalled
    HAS_IMAGEMAGICK = False

def protect_api(uuid=None, **kwargs):
    """Retrieve object and check permissions.

    Retrieve ObjectVersion of image being requested and check permission
    using the Invenio-Files-REST permission factory.
    """
    bucket, version_id, key = uuid.split(':', 2)
    g.obj = ObjectResource.get_object(bucket, key, version_id)
    return g.obj

def image_opener(key):
    """Handler to locate file based on key.

    .. note::
        If the file is a PDF then only the first page will be
        returned as an image.

    :param key: A key encoded in the format "<bucket>:<version>:<object_key>".
    :returns: A file-like object.
    """
    if hasattr(g, 'obj'):
        obj = g.obj
    else:
        obj = protect_api(key)

    fp = obj.file.storage().open('rb')

    # If ImageMagick with Wand is installed, extract first page
    # for PDF/text.
    if HAS_IMAGEMAGICK and obj.mimetype in ['application/pdf', 'text/plain']:
        first_page = Image(Image(fp).sequence[0])
        tempfile_ = tempfile.TemporaryFile()
        with first_page.convert(format='png') as converted:
            converted.save(file=tempfile_)
        return tempfile_
    return fp


from .constants import *
from .lexicon import *
from .glossary import *
from .realizer import *
from .script import *
from .utils import *

class parser():

    def __init__(self, lexicon, glossary, verbose = False):
        self.lang = None
        self.lexicon = dict()
        self.glossary = dict()
        self.verbose = verbose

        for fn in lexicon:
            load_lexicon(self.lexicon, fn)

        for fn in glossary:
            load_glossary(self.glossary, fn)

    def read(self, script):
        return load_script(script)

    def parse_node(self, idx, tree, feats):
        e = tree[idx]

        if e.state:
            return
        e.state = True

        if not e.feats:

            if e.head not in (-1, "*"): # if head exists
                self.parse_node(e.head, tree, feats)
                feats = avm(tree[e.head].feats)

            if e.head != -1:
                is_head = (e.head == "*")
                e.form, e.feats = realize(self, e.form, feats, is_head)

        if idx < len(tree) - 1:
            self.parse_node(idx + 1, tree, feats)

    def parse_term(self, idx, tree):
        e = tree[idx]

        if e.state:
            return
        e.state = True

        if type(e.feats) != avm:

            if e.head >= 0: # if head exists
                pass # TODO

            e.form = [node(i, *x) for i, x in enumerate(e.form)]
            e.feats = avm(e.feats)
            self.parse_node(0, e.form, e.feats)

        if idx < len(tree) - 1:
            self.parse_term(idx + 1, tree)

    def parse(self, text, terms):

        for lang, src in text:
            self.lang = lang

            # term realization

            tree = list()
            for i, m in enumerate(re.finditer(RE_TERM, src)):
                term, head, feats = m.groups()
                term = terms[int(term)] if term.isdecimal() else (*term.split(" "),)
                head = int(head) if head else -1

                if term in self.glossary:
                    lemma = self.glossary[term][lang][1]
                elif len(term) == 1 and term[0] in self.lexicon[lang][1]:
                    lemma = (("*", term[0]),)
                else:
                    sys.exit(ERR_LEMMA_NOT_FOUND % term)

                feats = feats[1:].split(":") if feats else None
                tree.append(node(i, head, lemma, feats, m.span()))

            self.parse_term(0, tree)

            k = 0
            tgt = src
            for e in tree:
                i, j = e.span
                term = " ".join(x.form for x in e.form)
                tgt = tgt[:i + k] + term + tgt[j + k:]
                k += len(term) - (j - i)

            # function word realization

            k = 0
            for m in re.finditer(RE_WORD, tgt):
                i, j = m.start() + k, m.end() + k
                word = realize_ko_morpheme(tgt[i - 1], m.group(1))
                tgt = tgt[:i] + word + tgt[j:]
                k += len(word) - (j - i)

            if self.verbose:
                printl("src =", src)
                printl("tgt =", tgt)
                printl("lang =", lang)
                for i, term in enumerate(terms):
                    if term in self.glossary:
                        term = self.glossary[term][lang][0]
                        printl("term[%d] =" % i, term)
                printl("tree =")
                for e in tree:
                    printl(e)
                input() # printl()

            yield src, tgt, tree


p=1000
r=0.07
n=10

tenth_year=1000*(1+0.07)*10
tenth_year=int(tenth_year)

print("the value of the tenth year is:  ",tenth_year)

p=1000
r=0.07
n=20

twenty_year=1000*(1+0.07)*20
twenty_year=int(twenty_year)

print("the value of the twentest year is:  ",twenty_year)

p=1000
r=0.07
n=30

thirtest_year=1000*(1+0.07)*30
thirtest_year=int(thirtest_year)

print("the value of the thirtest year is:  ",thirtest_year)


# 영화감독 숌
import sys
input = sys.stdin.readline
N = int(input())
number = []
flag = True
cur = 1
while flag:
    if str(cur).find('666') != -1:
        number.append(cur)
    cur += 1
    if len(number) >= N:
        flag = False

print(number[N-1])


import pessoa as p
import folhaPagamento as folha
import erros as e
import banco_de_dados as db

class funcionario(p.pessoa):
    def __init__(self):
        super().__init__()
        self.matriculaFuncionario = 0
        self.cargo = ""
        self.setor = ""
        self.cpf = 0
        self.salario_mes = 0
        self.nivel = 0
        self.salario_liquido = 0
    def cadastrar_funcionario(self):
        super().cadastrar_pessoa()
        listasMatricula = [x.matriculaFuncionario for x in db.DataBase('db/funcionario.pickle').loadObject()]
        msg = "Informe a matricula do funcionário: "
        self.matriculaFuncionario = e.tratarExistenciaPositiva(msg, listasMatricula)
        self.cargo = input("Informe o cargo: ")
        self.setor = input("Informe o setor: ")
        self.cpf = input("Informe o CPF: ")
        msg = "Informe o nível (1 a 4): "
        self.nivel = e.tratarNivel(msg)
        self.calcular_salario_func()
        inss = round(folha.folhaPagamento(self.salario_mes).calcular_inss(), 2)
        irrf = round(folha.folhaPagamento(self.salario_mes).calcular_irrf(), 2)
        salario_liquido = round(self.salario_liquido, 2)
        new = {'matricula_funcionario': self.matriculaFuncionario, 'cargo': self.cargo, 'setor': self.setor, 'cpf': self
            .cpf, 'nivel': self.nivel, 'salario_bruto': self.salario_mes, 'inss': inss , 'irrf': irrf, 'salario_liquido': salario_liquido}
        self.pessoa.update(new)
        print("")
        print("FUNCIONÁRIO CADASTRADO COM SUCESSO!!!")
        print("")
    def calcular_salario_func(self):
        if self.nivel == 1:
            self.salario_mes = 1500
        elif self.nivel == 2:
            self.salario_mes = 2380
        elif self.nivel == 3:
            self.salario_mes = 3500
        elif self.nivel == 4:
            self.salario_mes = 4277.32
        self.salario_liquido = folha.folhaPagamento(self.salario_mes).calcular_salario_liquido()
    def exibir_funcionario(self):
        return self.pessoa


"""
Dual Path Network as discriminator
"""
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import DataLoader, Dataset
from torch.nn.modules.distance import CosineSimilarity
import torchvision
from PIL import Image

import os
import sys
import numpy as np

# module definition - basic building block for DPN
class Bottleneck(nn.Module):
    def __init__(self, previous, first, last, increment, stride=1, is_first_layer=True, negative_slope=0.2):
        outf = last+increment
        super(Bottleneck, self).__init__()
        self.last = last
        self.layers = []
        self.layers.append(nn.Conv2d(in_channels=previous, out_channels=first, kernel_size=1, bias=False))
        self.layers.append(nn.BatchNorm2d(first))
        self.layers.append(nn.LeakyReLU(negative_slope=negative_slope))
        self.layers.append(nn.Conv2d(in_channels=first, out_channels=first, kernel_size=3,
                                     groups=32, stride=stride, padding=1, bias=False))
        self.layers.append(nn.BatchNorm2d(first))
        self.layers.append(nn.LeakyReLU(negative_slope=negative_slope))
        self.layers.append(nn.Conv2d(in_channels=first, out_channels=outf, kernel_size=1, bias=False))
        self.layers.append(nn.BatchNorm2d(outf))
        self.layers = nn.Sequential(*self.layers)

        self.shortcut = nn.Sequential()  # residual path
        # add one layer for possible size change during residual connection
        if is_first_layer:
            self.shortcut = nn.Sequential(
                nn.Conv2d(previous, outf, kernel_size=1, stride=stride, bias=False),
                nn.BatchNorm2d(outf)
            )
        self.outlayer = nn.LeakyReLU(negative_slope=negative_slope)

    def forward(self, x):
        densepath = self.layers(x)
        respath = self.shortcut(x)
        out = torch.cat([respath[:, :self.last, :, :]+densepath[:, :self.last, :, :],
                            respath[:, self.last:, :, :], densepath[:, self.last:, :, :]], dim=1)
        out = self.outlayer(out)
        return out

# model definition - DualPathNet
class DualPathNet(nn.Module):
    """
    General class to build DPN like network.
    Input: num_feats - color channels (3) of input images
           num_classes - number of classes (2300 for classification)
           structure - dictionary variable for network architecture
           task - either 'classify' or 'verify', specify the current task.
    """
    def __init__(self, num_feats, num_classes, structure):
        # input parsing
        in_sizes = structure['ins']
        out_sizes = structure['outs']
        block_repeats = structure['repeats']
        incre_sizes = structure['increments']
        init_hidden = structure['initial']
        k = structure['kernel']
        stride = structure['stride']
        negative_slope = structure['negative_slope']
        # initialization
        super(DualPathNet, self).__init__()
        self.prev = init_hidden

        # build network, input image size 32 by 32
#        self.padding = nn.modules.ConstantPad2d(image_height//2, 0)  # pad first to get to 64 by 64

        # first block is fully conv2d, max pooling is removed since input image is already small
        self.firstblock = []
        self.firstblock.append(nn.Conv2d(in_channels=num_feats, out_channels=init_hidden, padding=k//2, kernel_size=k,
                               stride=1, bias=False))  # original paper uses stride=2, use 1 here because of small image size
        self.firstblock.append(nn.BatchNorm2d(num_features=init_hidden))
        self.firstblock.append(nn.LeakyReLU(negative_slope=negative_slope))
#        self.firstblock.append(nn.MaxPool2d(3))  # max pooling used in original paper, may not be needed
        self.firstblock = nn.Sequential(*self.firstblock)

        # Bottleneck blocks
        self.layers = []
        for i in range(len(in_sizes)):
            repeat = block_repeats[i]
            first = in_sizes[i]
            last = out_sizes[i]
            increment = incre_sizes[i]
            self.layers.extend(self.conv_layer(first, last, increment, repeat, stride))
        self.layers = nn.Sequential(*self.layers)

        # fully connected layers
        last_out = out_sizes[-1] + (block_repeats[-1]+1)* incre_sizes[-1]
        self.fc_layers = nn.Linear(last_out, num_classes, bias=False)
        self.lastsig = nn.Sigmoid()

    def conv_layer(self, first, last, increment, repeat, stride):
        strides = [stride] + [1] * (repeat - 1)
        block = []
        for i, s in enumerate(strides):
            block.append(Bottleneck(self.prev, first, last, increment, stride=s, is_first_layer= (i==0)))
            self.prev = last + (i+2) * increment
        return block

    def forward(self, x, evalMode=False):
#        output = self.padding(x)
        output = self.firstblock(x)
        output = self.layers(output)

        # pooling layer is here
        output = F.avg_pool2d(output, [output.size(2), output.size(3)], stride=1)
        output = output.reshape(output.shape[0], output.shape[1])

        output = self.fc_layers(output)
        output = self.lastsig(output)

        return output

# weight initialization function
def init_weights(m, mode='xavier'):
    if type(m) == nn.Conv2d or type(m) == nn.Linear:
        if mode == 'xavier':
            torch.nn.init.xavier_normal_(m.weight.data)
        elif mode == 'he':
            torch.nn.init.kaiming_normal_(m.weight.data, nonlinearity='relu')

# macro for several DPN architecture
def DPNmini(num_feats, num_classes, kernel=3, stride=2, negative_slope=0.2):
    structure = {}
    structure['ins'] = [96, 128]
    structure['outs'] = [256, 512]
    structure['repeats'] = [2, 2, 2]
    structure['increments'] = [16, 24]
    structure['initial'] = 64
    structure['kernel'] = kernel
    structure['stride'] = stride
    structure['negative_slope'] = negative_slope

    return DualPathNet(num_feats, num_classes, structure)

def DPN26(num_feats, num_classes, kernel=3, stride=2, task='classify'):
    structure = {}
    structure['ins'] = [96, 192, 384, 768]
    structure['outs'] = [256, 512, 1024, 2048]
    structure['repeats'] = [2, 2, 2, 2]
    structure['increments'] = [16, 32, 24, 128]
    structure['initial'] = 64
    structure['kernel'] = kernel
    structure['stride'] = stride

    return DualPathNet(num_feats, num_classes, structure, task=task)

def DPN26small(num_feats, num_classes, kernel=3, stride=2, task='classify'):
    structure = {}
    structure['ins'] = [96, 384, 768]
    structure['outs'] = [256, 1024, 2048]
    structure['repeats'] = [2, 2, 2]
    structure['increments'] = [16, 24, 128]
    structure['initial'] = 64
    structure['kernel'] = kernel
    structure['stride'] = stride

    return DualPathNet(num_feats, num_classes, structure, task=task)

def DPN50(num_feats, num_classes, kernel=3, stride=2, task='classify'):
    structure = {}
    structure['ins'] = [96, 192, 384, 768]
    structure['outs'] = [256, 512, 1024, 2048]
    structure['repeats'] = [3, 4, 6, 3]
    structure['increments'] = [16, 32, 24, 128]
    structure['initial'] = 64
    structure['kernel'] = kernel
    structure['stride'] = stride

    return DualPathNet(num_feats, num_classes, structure, task=task)

def DPN50small(num_feats, num_classes, kernel=3, stride=2, task='classify'):
    structure = {}
    structure['ins'] = [192, 384, 768]
    structure['outs'] = [512, 1024, 2048]
    structure['repeats'] = [3, 6, 4]
    structure['increments'] = [32, 24, 128]
    structure['initial'] = 64
    structure['kernel'] = kernel
    structure['stride'] = stride

    return DualPathNet(num_feats, num_classes, structure, task=task)

def DPN92(num_feats, num_classes, kernel=3, stride=2, task='classify'):
    structure = {}
    structure['ins'] = [96, 192, 384, 768]
    structure['outs'] = [256, 512, 1024, 2048]
    structure['repeats'] = [3, 4, 20, 3]
    structure['increments'] = [16, 32, 24, 128]
    structure['initial'] = 64
    structure['kernel'] = kernel
    structure['stride'] = stride

    return DualPathNet(num_feats, num_classes, structure, task=task)

# function to remove '._' files
def findNremove(path,pattern,maxdepth=1):
    cpath=path.count(os.sep)
    for r,d,f in os.walk(path):
        if r.count(os.sep) - cpath <maxdepth:
            for files in f:
                if files.startswith(pattern):
                    try:
                        #print "Removing %s" % (os.path.join(r,files))
                        os.remove(os.path.join(r,files))
                    except Exception as e:
                        print(e)
                    else:
                        print("{} removed".format(os.path.join(r,files)))


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('staff', '0016_member'),
    ]

    operations = [
        migrations.DeleteModel(
            name='USAddress',
        ),
        migrations.AlterField(
            model_name='membershipdate',
            name='membership_date',
            field=models.DateField(unique=True),
        ),
    ]


########################################################################
############################# ASSIGNMENT 1 #############################
################################ Task 1 ################################
########################################################################

# %% Importing Libraries

import os
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

# %% Problem 1.1

# Changing the working directory
os.chdir("/Users/heinzemax/Documents/GitHub/inequality_sse/assignment_1")

# Reading in the data
income_data = pd.read_stata("/Users/heinzemax/Documents/GitHub/inequality_sse/assignment_1/psid1999.dta")

# Filtering the data so that only household heads are contained.
# Note that filtering by relationhead==10 is kind of a "best guess" after
# consulting PSID codebooks available online, since the variable in question
# is not labeled. Source for the "best guess": Page 4 of the document available at
# https://psidonline.isr.umich.edu/documents/psid/codebook/MX19REL_codebook.pdf
income_data = income_data.query("relationhead == 10")

# %% Problem 1.2

# Filtering the data set by age older than 24 and younger than 66
income_data = income_data.query("age > 24 & age < 66")

# Filtering the data set by married HHH with spouse present
income_data = income_data.query("(mls=='married' & sp_in_fu=='Spouse/Partner in FU now')")

# %% Problem 1.3

# Filtering out all household heads who worked zero hours
income_data = income_data.query("head_hrs > 0")

# Creating a logged hourly earnings variable from yearly income and yearly hrs,
# adding 1 to avoid taking the log of 0
income_data['log_hr_earnings'] = np.log((income_data['head_li'] / income_data['head_hrs']) + 1)

# Computing averages by age and education
average_wages = income_data.groupby(['age', 'edu'])['log_hr_earnings'].mean().reset_index()

# %% Problem 1.4

# Create a scatter plot showing avg log hrly earnings by age, with education
# levels represented as different colored dots
plt.figure(figsize=(8,6), dpi = 300)

education_levels = average_wages['edu'].unique()

for this_education_level in education_levels:
    subset = average_wages[average_wages['edu'] == this_education_level]
    plt.scatter(subset['age'], subset['log_hr_earnings'], label=this_education_level)

plt.title('Average Log Hourly Earnings by Age and Education')
plt.xlabel('Age')
plt.ylabel('Average Log Hourly Earnings')
plt.legend(title = "Education")
plt.grid(True, which='both', linestyle='--', linewidth=0.5)
plt.show()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models

class Migration(migrations.Migration):

    dependencies = [
        ('grafiks', '0016_remove_planotajs_one'),
    ]

    operations = [
        migrations.AlterField(
            model_name='grafiks',
            name='vietas',
            field=models.IntegerField(default=20),
        ),
    ]


from __future__ import print_function
from __future__ import unicode_literals
from localization import N_
from outputable import Outputable
import extensions
import filtering
import format
import gravatar
import interval
import os
import subprocess
import terminal
import textwrap

class FileDiff:
	def __init__(self, string):
		commit_line = string.split("|")

		if commit_line.__len__() == 2:
			self.name = commit_line[0].strip()
			self.insertions = commit_line[1].count("+")
			self.deletions = commit_line[1].count("-")

	@staticmethod
	def is_filediff_line(string):
		string = string.split("|")
		return string.__len__() == 2 and string[1].find("Bin") == -1 and ('+' in string[1] or '-' in string[1])

	@staticmethod
	def get_extension(string):
		string = string.split("|")[0].strip().strip("{}").strip("\"").strip("'")
		return os.path.splitext(string)[1][1:]

	@staticmethod
	def get_filename(string):
		return string.split("|")[0].strip().strip("{}").strip("\"").strip("'")

	@staticmethod
	def is_valid_extension(string):
		extension = FileDiff.get_extension(string)

		for i in extensions.get():
			if extension == i:
				return True
		return False

class Commit:
	def __init__(self, string):
		self.filediffs = []
		commit_line = string.split("|")

		if commit_line.__len__() == 4:
			self.date = commit_line[0]
			self.sha = commit_line[1]
			self.author = commit_line[2].strip()
			self.email = commit_line[3].strip()

	def add_filediff(self, filediff):
		self.filediffs.append(filediff)

	def get_filediffs(self):
		return self.filediffs

	@staticmethod
	def get_author_and_email(string):
		commit_line = string.split("|")

		if commit_line.__len__() == 4:
			return (commit_line[2].strip(), commit_line[3].strip())

	@staticmethod
	def is_commit_line(string):
		return string.split("|").__len__() == 4

class AuthorInfo:
	email = None
	insertions = 0
	deletions = 0
	commits = 0

class Changes:
	authors = {}
	authors_dateinfo = {}
	authors_email = {}

	def __init__(self, hard):
		self.commits = []
		git_log_r = subprocess.Popen("git log --pretty=\"%cd|%H|%aN|%aE\" --stat=100000,8192 --no-merges -w " +
		                             interval.get_since() + interval.get_until() +
		                             "{0} --date=short".format("-C -C -M" if hard else ""),
		                             shell=True, bufsize=1, stdout=subprocess.PIPE).stdout
		commit = None
		found_valid_extension = False
		lines = git_log_r.readlines()

		for i in lines:
			j = i.strip().decode("unicode_escape", "ignore")
			j = j.encode("latin-1", "replace")
			j = j.decode("utf-8", "replace")

			if Commit.is_commit_line(j):
				(author, email) = Commit.get_author_and_email(j)
				self.authors_email[author] = email

			if Commit.is_commit_line(j) or i is lines[-1]:
				if found_valid_extension:
					self.commits.append(commit)

				found_valid_extension = False
				commit = Commit(j)

			if FileDiff.is_filediff_line(j) and not filtering.set_filtered(FileDiff.get_filename(j)):
				extensions.add_located(FileDiff.get_extension(j))

				if FileDiff.is_valid_extension(j):
					found_valid_extension = True
					filediff = FileDiff(j)
					commit.add_filediff(filediff)

		if interval.has_interval() and len(self.commits) > 0:
			interval.set_ref(self.commits[0].sha)

	def get_commits(self):
		return self.commits

	def __modify_authorinfo__(self, authors, key, commit):
		if authors.get(key, None) == None:
			authors[key] = AuthorInfo()

		if commit.get_filediffs():
			authors[key].commits += 1

		for j in commit.get_filediffs():
			authors[key].insertions += j.insertions
			authors[key].deletions += j.deletions

	def get_authorinfo_list(self):
		if not self.authors:
			for i in self.commits:
				self.__modify_authorinfo__(self.authors, i.author, i)

		return self.authors

	def get_authordateinfo_list(self):
		if not self.authors_dateinfo:
			for i in self.commits:
				self.__modify_authorinfo__(self.authors_dateinfo, (i.date, i.author), i)

		return self.authors_dateinfo

	def get_author_email(self, name):
		return self.authors_email[name]

__changes__ = None

def get(hard):
	global __changes__
	if __changes__ == None:
		__changes__ = Changes(hard)

	return __changes__

HISTORICAL_INFO_TEXT = N_("The following historical commit information, by author, was found in the repository")
NO_COMMITED_FILES_TEXT = N_("No commited files with the specified extensions were found")

class ChangesOutput(Outputable):
	def __init__(self, hard):
		self.changes = get(hard)
		Outputable.__init__(self)

	def output_html(self):
		authorinfo_list = self.changes.get_authorinfo_list()
		total_changes = 0.0
		changes_xml = "

"
		chart_data = ""

		for i in authorinfo_list:
			total_changes += authorinfo_list.get(i).insertions
			total_changes += authorinfo_list.get(i).deletions

		if authorinfo_list:
			changes_xml += "
" + _(HISTORICAL_INFO_TEXT) + ".

"
			changes_xml += "

 {0}	 {1}	 {2}	 {3}	 {4}	".format(
			               _("Author"), _("Commits"), _("Insertions"), _("Deletions"), _("% of changes"))
			changes_xml += "

"

			for i, entry in enumerate(sorted(authorinfo_list)):
				authorinfo = authorinfo_list.get(entry)
				percentage = 0 if total_changes == 0 else (authorinfo.insertions + authorinfo.deletions) / total_changes * 100

				changes_xml += "
" if i % 2 == 1 else ">")

				if format.get_selected() == "html":
					changes_xml += "{1}	".format(
					               gravatar.get_url(self.changes.get_author_email(entry)), entry)
				else:
					changes_xml += "" + entry + "	"

				changes_xml += "" + str(authorinfo.commits) + "	"
				changes_xml += "" + str(authorinfo.insertions) + "	"
				changes_xml += "" + str(authorinfo.deletions) + "	"
				changes_xml += "" + "{0:.2f}".format(percentage) + "	"
				changes_xml += "
"
				chart_data += "{{label: \"{0}\", data: {1}}}".format(entry, "{0:.2f}".format(percentage))

				if sorted(authorinfo_list)[-1] != entry:
					chart_data += ", "

			changes_xml += ("

")
			changes_xml += "

"
			changes_xml += ""
		else:
			changes_xml += "
" + _(NO_COMMITED_FILES_TEXT) + ".
"

		changes_xml += "

"
		print(changes_xml)

	def output_text(self):
		authorinfo_list = self.changes.get_authorinfo_list()
		total_changes = 0.0

		for i in authorinfo_list:
			total_changes += authorinfo_list.get(i).insertions
			total_changes += authorinfo_list.get(i).deletions

		if authorinfo_list:
			print(textwrap.fill(_(HISTORICAL_INFO_TEXT) + ":", width=terminal.get_size()[0]) + "\n")
			terminal.printb(_("Author").ljust(21) + _("Commits").rjust(13) + _("Insertions").rjust(14) +
			                _("Deletions").rjust(15) + _("% of changes").rjust(16))

			for i in sorted(authorinfo_list):
				authorinfo = authorinfo_list.get(i)
				percentage = 0 if total_changes == 0 else (authorinfo.insertions + authorinfo.deletions) / total_changes * 100

				print(i.ljust(20)[0:20], end=" ")
				print(str(authorinfo.commits).rjust(13), end=" ")
				print(str(authorinfo.insertions).rjust(13), end=" ")
				print(str(authorinfo.deletions).rjust(14), end=" ")
				print("{0:.2f}".format(percentage).rjust(15))
		else:
			print(_(NO_COMMITED_FILES_TEXT) + ".")

	def output_xml(self):
		authorinfo_list = self.changes.get_authorinfo_list()
		total_changes = 0.0

		for i in authorinfo_list:
			total_changes += authorinfo_list.get(i).insertions
			total_changes += authorinfo_list.get(i).deletions

		if authorinfo_list:
			message_xml = "\t\t<message>" + _(HISTORICAL_INFO_TEXT) + "</message>\n"
			changes_xml = ""

			for i in sorted(authorinfo_list):
				authorinfo = authorinfo_list.get(i)
				percentage = 0 if total_changes == 0 else (authorinfo.insertions + authorinfo.deletions) / total_changes * 100
				name_xml = "\t\t\t\t<name>" + i + "</name>\n"
				gravatar_xml = "\t\t\t\t<gravatar>" + gravatar.get_url(self.changes.get_author_email(i)) + "</gravatar>\n"
				commits_xml = "\t\t\t\t<commits>" + str(authorinfo.commits) + "</commits>\n"
				insertions_xml = "\t\t\t\t<insertions>" + str(authorinfo.insertions) + "</insertions>\n"
				deletions_xml = "\t\t\t\t<deletions>" + str(authorinfo.deletions) + "</deletions>\n"
				percentage_xml = "\t\t\t\t<percentage-of-changes>" + "{0:.2f}".format(percentage) + "</percentage-of-changes>\n"

				changes_xml += ("\t\t\t<author>\n" + name_xml + gravatar_xml + commits_xml + insertions_xml +
				                deletions_xml + percentage_xml + "\t\t\t</author>\n")

			print("\t<changes>\n" + message_xml + "\t\t<authors>\n" + changes_xml + "\t\t</authors>\n\t</changes>")
		else:
			print("\t<changes>\n\t\t<exception>" + _(NO_COMMITED_FILES_TEXT) + "</exception>\n\t</changes>")