__author__ = 'suixin'

import sys
import threading
import socketserver
import numpy as np
import cv2
from predict import predict2
from carCtrHelper import RCTest

# distance data measured by ultrasonic sensor
sensor_data = None

class SensorDataHandler(socketserver.BaseRequestHandler):

    data = " "

    def handle(self):
        global sensor_data
        while self.data:
            self.data = self.request.recv(1024)
            sensor_data = round(float(self.data), 1)
            # print "{} sent:".format(self.client_address[0])
            print(sensor_data)

class VideoStreamHandler(socketserver.StreamRequestHandler):
    car = RCTest()
    count =0
    def handle(self):

        global sensor_data
        stream_bytes = b' '

        try:
            # stream video frames one by one
            while True:
                stream_bytes += self.rfile.read(1024)
                first = stream_bytes.find(b'\xff\xd8')
                last = stream_bytes.find(b'\xff\xd9')
                if first != -1 and last != -1:
                    jpg = stream_bytes[first:last + 2]
                    stream_bytes = stream_bytes[last + 2:]
                    #gray = cv2.imdecode(np.frombuffer(jpg, dtype=np.uint8), cv2.IMREAD_GRAYSCALE)
                    image = cv2.imdecode(np.frombuffer(jpg, dtype=np.uint8), cv2.IMREAD_COLOR)
                    cv2.imshow('image', image)
                    if cv2.waitKey(1) & 0xFF == ord('q'):
                        print("car stopped")
                        break
                    self.count = self.count+1
                    while self.count>50:
                        self.count = 0
                        #predict
                        direction  = predict2(image)
                        self.car.steer(direction)

        finally:
            cv2.destroyAllWindows()
            sys.exit()

class Server(object):
    def __init__(self, host, port1, port2):
        self.host = host
        self.port1 = port1
        self.port2 = port2

    def video_stream(self, host, port):
        s = socketserver.TCPServer((host, port), VideoStreamHandler)
        s.serve_forever()

    def sensor_stream(self, host, port):
        s = socketserver.TCPServer((host, port), SensorDataHandler)
        s.serve_forever()

    def start(self):
        sensor_thread = threading.Thread(target=self.sensor_stream, args=(self.host, self.port2))
        sensor_thread.daemon = True
        sensor_thread.start()
        self.video_stream(self.host, self.port1)

if __name__ == '__main__':
    h, p1, p2 = "192.168.0.104", 8000, 8002

    ts = Server(h, p1, p2)
    ts.start()


from requests import get
from bs4 import BeautifulSoup as bs
from threading import Thread

print('K O N T O O O O L'.center(40))
print('*'*50)
link = ['https://ailisgarcia.com/wp-content/uploads/2018/09/',
	  'https://ailisgarcia.com/wp-content/uploads/2018/10/',
	  'https://ailisgarcia.com/wp-content/uploads/2018/11/',
	  'https://ailisgarcia.com/wp-content/uploads/2018/12/']

o = open('hasil.txt','a')
bc = open('hasil.txt','r').read()
def gett():
  #for l1 in link:
	print('oke')
	r1 = get(z)
	b1 = bs(r1.text,'html.parser')
	a1 = b1.find_all('a')
	for li in a1:
		if '.txt' in li.get('href'):
			txt = get(z+li.get('href'))
			tt = txt.text
			if 'PW' in tt:#'EMAIL' in tt or 'PW' in tt or 'email' in tt or 'password' in tt:
				if tt in bc:
					pass
				print(tt)
				o.write(tt)

def trd():
  global z
  t =[]
  for z in link:
  	tx = Thread(name='heh',target=gett)
  	tx.start()
  	t.append(tx)
  for xxx in t:
  	xxx.join()
trd()
o.close()


import os
from pytube import YouTube
from tkinter import *
location = []
def download(youtube_link, download_folder):
    YouTube(youtube_link).streams.get_highest_resolution().download(download_folder)
    #yt.streams.filter(progressive=True, file_extension='mp4').order_by('resolution')[-1].download()
    #yt = YouTube('http://youtube.com/watch?v=9bZkp7q19f0')
    #print(yt.streams)
    return True

def clicked():

    #lbl.configure(text="Button was clicked !!")
    res =  txt.get()
    lbl_1.configure(text = "Please wait")
    #lbl.configure(text= res)
    r = download(res, location)
    if r is True:
        lbl_1.configure(text = "done")
    else:
        return False

############## Get user home.
home = os.path.expanduser('~')
print(home)
location = os.path.join(home, 'Downloads')

### Start GUI.
window = Tk()

window.title("Download App")

window.geometry('1000x100')
lbl_0a = Label(window, text = "Download YouTube file", font=("Arial Bold", 10))
lbl_0a.grid(column=0, row= 0)
lbl_0b = Label(window, text = "Use Ctrl-C to copy link and Ctrl_V to paste", font=("Arial Bold", 10))
lbl_0b.grid(column=1, row= 0)

lbl = Label(window, text="Copy and Paste YouTube link:",   bg='white')

lbl.grid(column=0, row=1)

txt = Entry(window,width=100, text='Put link here')

txt.grid(column=1, row=1)

###btn = Button(window, text="Click Me")
btn = Button(window, text='Download',command=clicked)

btn.grid(column=1, row=2)
### Just show wait/done
lbl_1 = Label(window, text = "Wait")
lbl_1.grid(column=1, row = 3)

window.mainloop()


import pytest

from modbus_client.communication.serializer import *

protocol_code = '0000'

@pytest.mark.parametrize(['message', 'expected'],
                         [(bytes.fromhex('0005000000050201028002'), {'transaction_id': 5,
                                                                     'unit_address': 2,
                                                                     'function_code': 1,
                                                                     'status_list': [0, 0, 0, 0, 0, 0, 0, 1,
                                                                                0, 1, 0, 0, 0, 0, 0, 0],
                                                                     'raw_data': bytes.fromhex('028002')}),
                          (bytes.fromhex('000A000000050102020500'), {'transaction_id': 10,
                                                                     'unit_address': 1,
                                                                     'function_code': 2,
                                                                     'status_list': [1, 0, 1, 0, 0, 0, 0, 0,
                                                                                0, 0, 0, 0, 0, 0, 0, 0],
                                                                     'raw_data': bytes.fromhex('020500')}),
                          (bytes.fromhex('000F0000000701030403E81388'), {'transaction_id': 15,
                                                                         'unit_address': 1,
                                                                         'function_code': 3,
                                                                         'register_data': ['1000', '5000'],
                                                                         'raw_data': bytes.fromhex('0403E81388')}),
                          (bytes.fromhex('0014000000070104042710C350'), {'transaction_id': 20,
                                                                         'unit_address': 1,
                                                                         'function_code': 4,
                                                                         'register_data': ['10000', '50000'],
                                                                         'raw_data': bytes.fromhex('042710C350')}),
                          (bytes.fromhex('00190000000601050064FF00'), {'transaction_id': 25,
                                                                       'unit_address': 1,
                                                                       'function_code': 5,
                                                                       'raw_data': bytes.fromhex('0064FF00')}),
                          (bytes.fromhex('001E00000006010600643A98'), {'transaction_id': 30,
                                                                       'unit_address': 1,
                                                                       'function_code': 6,
                                                                       'raw_data': bytes.fromhex('00643A98')}),
                          (bytes.fromhex('0023000000061C1000640002'), {'transaction_id': 35,
                                                                       'unit_address': 28,
                                                                       'function_code': 16,
                                                                       'raw_data': bytes.fromhex('00640002')})])
def test_deserialize_message(message, expected):
    assert deserialize_message(message) == expected

@pytest.mark.parametrize(['function_code', 'transaction_id', 'unit_address', 'first_address', 'count', 'expected'],
                         [(1, 5, 2, 32, 12, '00050000000602010020000c'),
                          (2, 10, 1, 500, 32, '000a00000006010201f40020'),
                          (3, 15, 1, 600, 2, '000f00000006010302580002'),
                          (4, 20, 1, 200, 2, '001400000006010400c80002')])
def test_serialize_read(function_code, transaction_id, unit_address, first_address, count, expected):
    assert serialize_read(function_code, transaction_id, unit_address, first_address, count) == expected

@pytest.mark.parametrize(['transaction_id', 'unit_address', 'address', 'status', 'expected'],
                         [(25, 1, 100, True, '00190000000601050064FF00')])
def test_serialize_write_single_coil(transaction_id, unit_address, address, status, expected):
    assert serialize_write_single_coil(transaction_id, unit_address, address, status) == expected

@pytest.mark.parametrize(['transaction_id', 'unit_address', 'address', 'data', 'expected'],
                         [(30, 1, 100, 15000, '001e00000006010600643a98')])
def test_serialize_write_single_register(transaction_id, unit_address, address, data, expected):
    assert serialize_write_single_register(transaction_id, unit_address, address, data) == expected

@pytest.mark.parametrize(['transaction_id', 'unit_address', 'first_address', 'data', 'expected'],
                         [(1, 1, 19, ['1', '0', '1', '1', '0', '0', '1', '1', '1', '0'],
                           '000100000009010f0013000a02cd01')])
def test_serialize_write_multiple_coils(transaction_id, unit_address, first_address, data, expected):
    assert serialize_write_multiple_coils(transaction_id, unit_address, first_address, data) == expected

@pytest.mark.parametrize(['transaction_id', 'unit_address', 'first_address', 'data', 'expected'],
                         [(35, 28, 100, [1000, 2008], '00230000000b1c10006400020403e807d8')])
def test_serialize_write_multiple_registers(transaction_id, unit_address, first_address, data, expected):
    assert serialize_write_multiple_registers(transaction_id, unit_address, first_address, data) == expected


import itertools
import sys

test_cases = open(sys.argv[1], 'r')

def permute(the_string):
    # turn string into sorted array of chars
    arr = [c for c in the_string]
    arr.sort()
    # then get all possible permutations, itertools implementation
    # gives us the desired ordering, no more work to do
    return [''.join(p) for p in itertools.permutations(arr, len(arr))]

for test in test_cases:
    if test == "":
        continue
    permutations = permute(test.strip())
    print(','.join(permutations))


import pandas as pd
from pathlib import Path
from docxtpl import DocxTemplate
from datetime import datetime
from PyPDF2 import PdfReader, PdfWriter

#Directory Paths for each file
binder_binder_fee = [ "Binder Fee Invoice - Cedar.pdf",
                      "Binder.docx"
                    ]
cancel_letter = "Cancellation-Mid-Term-or-Flat Letter.docx"
base_dir = Path(__file__).parent.parent
excel_path = base_dir / "input.xlsx"  # name of excel
output_dir = base_dir / "output" # name of output folder
output_dir.mkdir(exist_ok=True)

#Pandas reading excel file
df = pd.read_excel(excel_path, sheet_name="Sheet3")

#Format date to MMM DD, YYYY
df["today"] = datetime.today().strftime("%B %d, %Y")
df["effective_date"] = df["effective_date"].dt.strftime("%B %d, %Y")

#Remove white spaces, capitalize first letter of each word, and format effective date time
def namesFormatter(df):
    for i in df.columns:
      if df[i].dtype == 'object':
        df[i] = df[i].str.strip()
      if df["policy_number"].dtype == 'object':
        df["policy_number"] = df["policy_number"].astype("string")
      if df["type"].dtype == 'object':
        df["type"] = df["type"].str.title()
      if df["insured_name"].dtype == 'object':
        df["insured_name"] = df["insured_name"].str.title()
      if df["insurer"].dtype == 'object':
        df["insurer"] = df["insurer"].str.title()
      if df["broker_name"].dtype == 'object':
        df["broker_name"] = df["broker_name"].str.title()
      else:
        pass
namesFormatter(df)

#Checks if there is additonal_insured
def insuredNames(rows):
  if (pd.isnull(rows["additional_insured"])):
    return rows["insured_name"]
  return rows["insured_name"] + " & " + rows['additional_insured']

#Checks if no risk address, use mailing address as risk address
def riskAddress(rows):
  if (pd.isnull(rows["risk_address"])):
    return rows["mailing_address"]
  return rows["risk_address"]

#Checks if there is a policy_number
def checkPolicyNumber(rows):
  if (pd.isnull(rows["policy_number"])):
    return ""
  return rows["policy_number"]

#Checks if there is a policy_number
def checkClientCode(rows):
  if (pd.isnull(rows["client_code"])):
    return ""
  return rows["client_code"]

#Checks if there is an effective date
def checkEffectiveDate(rows):
  if (pd.isnull(rows["effective_date"])):
    return ""
  return rows["effective_date"]

#Reads and writes PDF
def writeToPdf(pdf, dictionary, rows):
  pdf_path = base_dir / "input" / pdf
  reader = PdfReader(pdf_path)
  writer = PdfWriter()
  for pageNum in range(reader.numPages):
    page = reader.getPage(pageNum)
    writer.add_page(page)
  writer.updatePageFormFieldValues(
  writer.getPage(0), dictionary
  )
  output_path = output_dir / f"{insuredNames(rows)}" / f"{insuredNames(rows)} - {pdf}"
  output_path.parent.mkdir(exist_ok=True)
  with open(output_path, "wb") as output_stream:
    writer.write(output_stream)

#Write to Docx
def writeToDocx(docx, rows):
  template_path = base_dir / "input" / docx
  doc = DocxTemplate(template_path)
  doc.render(rows)
  output_path = output_dir / f"{insuredNames(rows)}" / f"{insuredNames(rows)} - {docx}"
  output_path.parent.mkdir(exist_ok=True)
  doc.save(output_path)

for rows in df.to_dict(orient="records"):
  # Make Cancel/Lapse Letter
  if (rows["type"] == "Cancel" or rows["type"] == "Lapse"):
    writeToDocx(cancel_letter, rows)
  #Binder Invoice and Binder
  if (rows["type"] == "Binder"):
    dictionary = {'Effective Date': checkEffectiveDate(rows),
                  'Policy Number' : checkPolicyNumber(rows),
                  'Account Number' : checkClientCode(rows),
                }
    writeToPdf(binder_binder_fee[0], dictionary, rows)
    writeToDocx(binder_binder_fee[1], rows)


from django.contrib import admin

from django.urls import path
from . import views

urlpatterns = [
path("",views.home,name="home"),
path("about",views.about,name="about"),
path("contact",views.contact,name="contact"),
path("onepiece",views.onepiece,name="onepiece"),
path("naruto",views.naruto,name="naruto"),
path("demonslayer",views.demonslayer,name="demonslayer"),

]


import time
from datetime import datetime

import psutil
from pyrogram import filters
from pyrogram.types import InlineKeyboardButton, InlineKeyboardMarkup, Message

import config
from FallenMusic import BOT_NAME, StartTime, app
from FallenMusic.Helpers import get_readable_time

@app.on_message(filters.command("ping"))
async def ping_fallen(_, message: Message):
    hmm = await message.reply_photo(
        photo=config.PING_IMG, caption=f"{BOT_NAME} ɪs ᴘɪɴɢɪɴɢ..."
    )
    upt = int(time.time() - StartTime)
    cpu = psutil.cpu_percent(interval=0.5)
    mem = psutil.virtual_memory().percent
    disk = psutil.disk_usage("/").percent
    start = datetime.now()
    resp = (datetime.now() - start).microseconds / 1000
    uptime = get_readable_time((upt))

    await hmm.edit_text(
        f"""➻ ᴩᴏɴɢ : `{resp}ᴍs`

{BOT_NAME} sɪsᴛᴇᴍ ɪsᴛᴀᴛɪsᴛɪᴋʟᴇʀɪ :

๏ **ᴜᴩᴛɪᴍᴇ :** {uptime}
๏ **ʀᴀᴍ :** {mem}
๏ **ᴄᴩᴜ :** {cpu}
๏ **ᴅɪsᴋ :** {disk}""",
        reply_markup=InlineKeyboardMarkup(
            [
                [
                    InlineKeyboardButton("❄ ᴅᴇsᴛᴇᴋ ❄", url=config.SUPPORT_CHAT),
                ],
            ]
        ),
    )


import gym
import argparse
from stable_baselines3 import DQN
from stable_baselines3 import PPO
from gym import spaces
from stable_baselines3.common.env_checker import check_env
from stable_baselines3.common.callbacks import StopTrainingOnMaxEpisodes
from stable_baselines3.common.evaluation import evaluate_policy

import host
import env_pb2
import ns3_mock as ns
import numpy as np
class ProtoHostEnv(gym.core.Env):
    def __init__(self, _host, port):
        self.iface = host.ProtoHost()
        self.iface.listen_and_accept(_host, port)
        super(ProtoHostEnv, self).__init__()
        self.action_space = spaces.Discrete(2)
        self.observation_space = spaces.Box(low=-10000.0, high=10000.0, shape= (11,),dtype=np.float32)
        self.num_episode=0

    def step(self, action):
        self.__send_action(action)
        state, reward, done, info = self.__rcv_env_state()
        return np.array(state), reward, done, dict(info)

    def render(self, mode="human", close="False"):
        return

    def close(self):
        pass

    def seed(self, seed=None):
        self.np_random, seed = gym.utils.seeding.np_random(seed)
        return [seed]

    def reset(self):
        state = self.__rcv_init_state()
        return np.array(state)

    def __rcv_env_state(self):
        s = self.iface.rcv_proto(env_pb2.EnvState, timeout=60)
        return s.state.value, s.reward, s.done, s.info

    def __rcv_init_state(self):
        s = self.iface.rcv_proto(env_pb2.State, timeout=60)
        return s.value

    def __send_action(self, a):
        action = env_pb2.Action()
        action.value = a
        self.iface.send_proto(action)

def parse_args():
    parser = argparse.ArgumentParser("ClientServerEnv")
    parser.add_argument(
            "--logging", type=str, default="INFO", help="Set log level")
    parser.add_argument(
            "--host",
            type=str,
            default="127.0.0.1",
            help="Host name")
    parser.add_argument(
            "--port", type=int, default=50001, help="Host port")
    args = parser.parse_args()
    assert args.port > 50000, "Port must be larger than 50000"
    return args

def main():
    args = parse_args()
    host.set_logging_config(args.logging)
    # Stops training when the model reaches the maximum number of episodes
    callback_max_episodes = StopTrainingOnMaxEpisodes(max_episodes=15000, verbose=1)
    env = ProtoHostEnv(args.host, args.port)
    #train a Agent#
    #model = DQN("MlpPolicy", env,learning_rate=0.001,learning_starts=1000,exploration_final_eps = 0.05,batch_size =128, gamma = 0.95, verbose=1)

    #model.learn(total_timesteps=900000, callback=callback_max_episodes, log_interval=2)

    #model.save("DQN_HO_4")
    #del model # remove to demonstrate saving and loading

    model = DQN.load("DQN_HO_4")
    s = env.reset()
    print(f"init state: {s}")
    for i in range(40000):
        action ,_states = model.predict(s,deterministic=True)

        s, reward, done, info = env.step(action)
        print(s, reward, done, info)
        if done:
            s = env.reset()

if __name__ == "__main__":
    main()


import numpy as np # linear algebra

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import cv2

import skimage

from skimage.filters import gaussian

import matplotlib.pyplot as plt

# Input data files are available in the "../input/" directory.

# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory

from subprocess import check_output

#print(check_output(["ls", "../input/stage1_train/00071198d059ba7f5914a526d124d28e6d010c92466da21d4a04cd5413362552/masks"]).decode("utf8"))

# Any results you write to the current directory are saved as output.
stage1_train_labels = pd.read_csv('../input/stage1_train_labels.csv')

stage1_train_labels.loc[0,'ImageId']
class image():

    def __init__(self, img_code, stage='stage1'):

        self.stage = stage

        self.img_code = img_code

        self.file_path = '../input/' + stage + '_train/' + img_code

        self.images_path = self.file_path + '/images'

        self.mask_path = self.file_path + '/masks'

        self.img = cv2.imread(self.images_path + '/' + self.img_code + '.png')

        self._mask_file_names = self.get_mask_file_names()

        self.n_nuclei = len(self._mask_file_names)

        self.masks = self.get_masks()

        self.mask = np.sum(self.masks,axis=0)

        self.mask_with_outline = self.combine_masks_with_outline()

    def get_mask_file_names(self):

        file_names_string = check_output(["ls", "../input/" + self.stage + "_train/" + self.img_code + "/masks"]).decode("utf8")

        mask_file_names = file_names_string.split('\n')

        return [mask_file_name for mask_file_name in mask_file_names if len(mask_file_name) > 0]

    def get_masks(self):

        masks = []

        for mask_file_name in self._mask_file_names:

            mask = cv2.imread(self.mask_path+'/'+mask_file_name, cv2.IMREAD_GRAYSCALE)

            masks += [(mask > 0).astype(int)]

        return masks

    def combine_masks_with_outline(self):

        combined = self.mask.copy()

        for mask in self.masks:

            outline = (np.abs((gaussian(mask,sigma=1,preserve_range=True) - mask))*20).astype(int)

            combined += outline

        return combined

im1 = image(stage1_train_labels.loc[13005,'ImageId'])

plt.imshow(im1.img)

plt.figure()

plt.imshow(im1.mask_with_outline)

im1.n_nuclei


'''
Let's see how clunky it is to define and run ltspice simulations from python.
Will be possible to express much more complicated operations than in the spice language

As far as I know, we must communicate with ltspice via file io

Just need to translate python into spice language.
More often we will input existing netlists and modify them.
I'm taking a ~functional programming approach for fun, but could imagine an object oriented way that could be ok, too.

There is a library by Nuno Brum called PyLTSpice, which I tested and it seems to work.
https://github.com/nunobrum/PyLTSpice
Problem is he wrote his own data container classes which are clunky and I don't want to use them.
I want to use built-in data classes, like dict and list
which can be converted into well established and maintained containers like pandas series and dataframes for further analysis.

There's also this, which I haven't tested
https://github.com/DongHoonPark/ltspice_pytool
'''
#TODO: Spice can't tolerate spaces in equations, which makes them really hard to read.
# It took me an hour to realize this, anything following a space is just ignored
# We could easily allow spaces here and then always delete them before writing to disk
#TODO: Spice is case insensitive -- change everything to account for that
#TODO: Spice has lots of different syntax for the same thing.  e.g. .param name value  .PARAM name=value ..
#      write a better parser

import subprocess
import os
import re
import numpy as np
from collections.abc import Iterable
from datetime import datetime
import time
import fnmatch
from numbers import Number
from functools import reduce, partial
import warnings
import hashlib

# ltspice uses whatever encoding it feels like using, needs to be detected
# I think it takes cues from what kind of characters you use in the GUI
import chardet

# Make sure this points to your spice executable, and that it is the XVII version
spicepath = r"C:\Program Files\LTC\LTspiceXVII\XVIIx64.exe"

# Here is where all the simulation files (netlists and results) will be dumped
# It can get quite large if you don't delete the files afterward
simfolder = r"ltspice_sims"
simfolder = os.path.abspath(simfolder)

if not os.path.isdir(simfolder):
    os.makedirs(simfolder)

net_hashes = dict()

# Sample netlist -- list of strings
netlist = '''
* Example netlist
V1 in 0 PULSE(0 5 1m 1n 1n 10m)
R1 in N001 {R}
L1 N001 N002 {L}
C1 N002 0 {C}
.PARAM C=1e-6
.PARAM L=1e-3
.PARAM R=1
.tran 0 10m 0
.backanno
.end
'''.strip().split('\n')

def read_and_decode(filepath):
    ''' Use this to read a file if you don't know what the encoding is '''
    with open(filepath, 'rb') as f:
        data = f.read()
    encoding = chardet.detect(data)['encoding']
    return data.decode(encoding)

### File IO
def netlist_fromfile(filepath):
    ''' Read ltspice .net file.  Return a list of strings'''
    netlist = read_and_decode(filepath).split('\n')
    return [nl.strip() for nl in netlist if nl]

def read_log(filepath):
    ''' Read ltspice .log file and parse out some information.  Return a dict'''
    filepath = replace_ext(filepath, 'log')
    print(f'Reading {filepath}')
    lines = read_and_decode(filepath)
    logitems = re.findall('(.[^\s]*)[=,:] (.*)', lines)
    if not logitems:
        # quick fix, don't know why, but sometimes we have utf-16 encoding and sometimes not
        # chardet can't seem to detect this
        with open(filepath, 'r', encoding='utf-16-le') as f:
            lines = f.read()
        logitems = re.findall('(.[^\s]*)[=,:] (.*)', lines)
    def convert_vals(val):
        val = val.strip()
        if val.isdigit():
            val = np.int32(val)
        return val
    # might have more than one of each type (e.g. multiple WARNINGs)
    # store them as lists of strings if there are multiple and regret it later
    logdict = {}
    for k,v in logitems:
        key = k.strip()
        if key in logdict:
            if not isinstance(logdict[key], list):
                logdict[key] = [logdict[key]]
            logdict[key].append(convert_vals(v))
        else:
            logdict[key] = convert_vals(v)
    if 'Total elapsed time' in logdict:
        logdict['sim_time'] = np.float32(logdict['Total elapsed time'].split(' ')[0])
        del logdict['Total elapsed time']
    return logdict

def read_net(filepath):
    ''' Read ltspice .net file and parse out some information.  Return a dict'''
    netinfo = {}
    filepath = replace_ext(filepath, 'net')
    print(f'Reading {filepath}')
    netlist = netlist_fromfile(filepath)
    netparams = get_params(netlist)
    #netfuncs = re.findall('.FUNC (.*)=(.*)', netlist)
    #netfuncs = {k:v for k,v in netfuncs}
    netinfo['netlist'] = netlist
    netinfo.update(netparams)
    return netinfo

def read_raw(filepath):
    '''
    Read ltspice output .raw file.
    return dict of parameters and arrays contained in the file
    for ltspice XVII version
    Does not load parameter runs because I think those should just be done in python
    '''
    filepath = os.path.abspath(filepath)
    filepath = replace_ext(filepath, 'raw')

    # Read information from the .raw file
    print(f'Reading {filepath}')
    with open(filepath, 'rb') as raw_file:
        colnames = []
        units = []
        raw_params = {'filepath': filepath}

        def readline():
            line = raw_file.readline()
            # Read the newline...
            raw_file.read(1)
            return line.decode(encoding='utf_16_le', errors='ignore')

        # Read the header
        line = readline()
        while not line.startswith('Variables:'):
            tag, value_str = line.split(':', 1)
            value_str = value_str.strip()
            if value_str.isdigit():
                value_str = int(value_str)
            raw_params[tag] = value_str
            line = readline()

        # Read the column names
        line = readline()
        while not line.startswith('Binary'):
            _, name, unit = line.strip().split('\t')
            colnames.append(name)
            units.append(unit)
            line = readline()

        # Read the numerical data
        # Time values are 8 bytes, rest are 4 bytes.
        # (raw_file_size - 930) / numpoints / ((numvars-1)*4 + 8) = 1
        numpoints = raw_params['No. Points']
        numvars = raw_params['No. Variables']
        dtype = np.dtype({'names':colnames,
                          'formats':[np.float64] + [np.float32]*(numvars - 1)})
        # d is a dreaded numpy structured array
        d = np.fromfile(raw_file, dtype)
        ddict = {k:d[k] for k in colnames}
        # I have no idea why this is necessary
        if 'time' in ddict:
            ddict['time'] = np.abs(ddict['time'])
        for k,v in ddict.items():
            if v.size == 1:
                # numpy 0d arrays are stupid
                ddict[k] = v.item()
    # Put data and metadata together in one dict
    outdict = {**raw_params, **ddict}

    return outdict

def read_spice(filepath, namemap=None):
    ''' Read all the information contained in all the spice files with the same name (.raw, .net, .log)'''
    filepath = os.path.abspath(filepath)

    rawdata = read_raw(filepath)
    logdata = read_log(filepath)
    netdata = read_net(filepath)

    # Pick and choose the data you want to output
    dataout = {**rawdata, **netdata}
    dataout['sim_time'] = logdata.get('sim_time')
    dataout['solver'] = logdata.get('solver')
    dataout['method'] = logdata.get('method')
    dataout['WARNING'] = logdata.get('WARNING')

    # Map to other names if you want
    if namemap is not None:
        for oldk, newk in namemap.items():
            if oldk in dataout:
                dataout[newk] = dataout[oldk]
                del dataout[oldk]

    return dataout

def replace_ext(path, newext):
    return os.path.splitext(path)[0] + '.' + newext.strip('.')

def timestamp():
    ''' Timestamp string with ms accuracy '''
    return datetime.now().strftime('%Y-%m-%d_%H%M%S_%f')[:-3]

def valid_filename(s):
    s = str(s).strip().replace(' ', '_')
    return re.sub(r'(?u)[^-\w.]', '', s)

def write_wav(times, voltages, filename):
    import struct

    def lin_interp(x0, x1, y0, y1, x):
        x0 = float(x0)
        x1 = float(x1)
        y0 = float(y0)
        y1 = float(y1)
        x = float(x)
        return y0 + (x - x0) * (y1 - y0) / (x1 - x0)

    with wave.open(filename, 'w') as w:
        w.setnchannels(1)
        w.setsampwidth(2)
        w.setframerate(44100)
        w.setnframes(0)
        w.setcomptype('NONE')
        w.writeframes(np.sin(np.linspace(0, 8000*2*pi, 100000)))
        m = max(max(voltages), -min(voltages))
        vrange = m

        values = bytes([])
        t = 0.0
        step = 1.0 / SAMPLING_RATE

        for i in range(len(voltages)-1):
            while times[i] <= t * step < times[i+1]:
                sample = lin_interp(times[i], times[i+1],
                    voltages[i], voltages[i+1], t*step) / vrange
                sample = 1 if sample >1 else sample
                sample = -1 if sample <-1 else sample
                d = struct.pack('<h',int(32767 * sample))
                values += d
                t += 1

        w.writeframes(values)

def hash(netlist):
    return hashlib.md5(bytes('\n'.join(netlist), 'utf-8')).hexdigest()

def from_cache(netlist, namemap=None):
    ''' Check whether the same netlist has already been run, and if yes, return the results written to disk'''
    # Update cache
    existing_fns = fnmatch.filter(os.listdir(simfolder), '*.net')
    for fn in existing_fns:
        if fn not in net_hashes.values():
            existing_net = netlist_fromfile(os.path.join(simfolder, fn))
            #net_hashes[fn] = hash(existing_net)
            # only add to cache if there is corresponding output data
            raw_exists = os.path.isfile(os.path.join(simfolder, fn[:-3] + 'raw'))
            log_exists = os.path.isfile(os.path.join(simfolder, fn[:-3] + 'log'))
            if raw_exists & log_exists:
                net_hashes[hash(existing_net)] = fn
    # Check if hash already in the cache
    h = hash(netlist)
    if h in net_hashes:
        print('Reading previous matching sim from disk')
        return read_spice(os.path.join(simfolder, net_hashes[h]), namemap=namemap)

    return False

# TODO somehow stop spice from stealing focus even though no window is visible
# TODO cache some inputs and outputs, so that you don't keep running the same simulations
# at least cache the file locations
#from functools import lru_cache
#@lru_cache(maxsize=32)
def runspice(netlist, namemap=None, timeout=None, check_cache=True):
    ''' Run a netlist with ltspice and return all the output data '''
    # TODO: Sometimes when spice has an error, python just hangs forever.  Need a timeout or something.
    # TODO: is there any benefit to spawning many ltspice processes at once, instead of sequentially?
    if type(netlist) is str:
        netlist = netlist.split('\n')
    if check_cache:
        old_result = from_cache(netlist, namemap=namemap)
        if old_result: return old_result
    t0 = time.time()
    # Write netlist to disk
    title = valid_filename(get_title(netlist))
    netlistfp = os.path.join(simfolder, timestamp() + f'_{title}.net')
    netlistfp = os.path.abspath(netlistfp)
    print(f'Writing {netlistfp}')
    with open(netlistfp, 'w') as f:
        f.write('\n'.join(netlist))
    print(f'Executing {netlistfp}')
    # Tell spice to execute it
    # If error/timeout, maybe we want to keep running things, don't raise the error just return empty data
    try:
        subprocess.check_output([spicepath, '-b', '-Run', netlistfp], timeout=timeout)
    except subprocess.CalledProcessError as err:
        print(err)
        print(read_log(replace_ext(netlistfp, 'log')))
        return {}
    except subprocess.TimeoutExpired as err:
        print(err)
        return {}

    #subprocess.check_output([spicepath, '-b', '-ascii', '-Run', netlistfp])
    # Read result
    rawfp = os.path.splitext(netlistfp)[0] + '.raw'
    d = read_spice(rawfp, namemap=namemap)
    t1 = time.time()
    # Sim time including file io
    d['sim_time_total'] = t1 - t0
    return d

def recentfile(filter='', n=0, folder=simfolder):
    ''' Return the nth most recent filepath'''
    filter = f'*{filter}*'
    dirlist = os.listdir(folder)
    matches = fnmatch.filter(dirlist, filter)
    matchingfps = [os.path.join(folder, m) for m in matches]
    # might be able to just assume they are in sorted order because of the file names...
    recent = np.argsort([os.path.getmtime(f) for f in matchingfps])
    return os.path.join(folder, matches[recent[-1-n]])

### Netlist parsing
def get_params(netlist):
    params = re.findall('.PARAM (.*)=(.*)', '\n'.join(netlist))
    params = {k:v for k,v in params}
    # Try to convert params to number
    for k,v in params.items():
        try:
            params[k] = np.float32(v)
        except ValueError:
            pass
    return params

def get_title(netlist):
    ''' My understanding is that the first line must always be the title '''
    if netlist[0].startswith('*'):
        title = netlist[0][1:].strip()
    else:
        title = netlist[0]
    return title

### Functions for operating on netlist
def netinsert(netlist, newline):
    '''
    Replace a line in the netlist if it corresponds to an existing command,
    or else add it as a new line in a reasonable position
    Does not check if you are putting in a command that makes sense or will run!
    Return the netlist with the inserted line
    '''
    # Spice language is not very uniform so it's not completely trivial how to decide
    # When to replace a line or when to add a new line.

    netlist = netlist.copy()

    # Find the part of the string that identifies what the command does
    cmd, *rest = newline.split(' ', 1)
    if cmd.lower() in ('.param', '.func', '.ic'):
        # Compare also with the thing before the = sign
        cmd_id = newline[:newline.find('=') + 1]
    else:
        cmd_id = cmd

    # How similar is each line to the identifying string?
    # I will put the new line after the one most similar to it
    # There's probably a better approach.
    def similarity(netlist, line):
        '''
        Return the number of characters that are the same as 'line' for each line of netlist
        '''
        # This is not the most efficient thing ever but who cares
        def compare(str1, str2):
            if str1.lower().startswith(str2.lower()):
                return len(str2)
            else:
                return [c1.lower() == c2.lower() for c1,c2 in zip(str1, str2)].index(False)
        return [compare(l, line) for l in netlist]

    simi = similarity(netlist, cmd_id)
    maxsim = max(simi)
    if maxsim == len(cmd_id):
        # Replace existing line
        i = simi.index(maxsim)
        netlist[i] = newline
    else:
        # Find a good index to insert the new line
        # (After the most similar line)
        i = np.argmax(simi) + 1
        netlist.insert(i, newline)

    return netlist

def netchange(netlist, *newlines):
    '''
    Pass any (potentially nested) list of strings and merge them with netlist
    Return the resulting merged netlist
    '''
    strings = flatten(newlines)
    return reduce(netinsert, strings, netlist)

def netchanger(netlist):
    ''' Closure that remembers the input netlist, and allows you to modify it by passing partial netlists '''
    changer = partial(netchange, netlist)
    return changer

def paramchange(netlist, paramdict=None, **kwargs):
    '''
    If you only want to change .PARAMS, you can pass a dict of them to this function
    Or pass them as keyword arguments, or both
    '''
    newparams = []
    if paramdict is not None:
        newparams += [param(k, v) for k,v in paramdict.items()]
    if kwargs:
        newparams += [param(k, v) for k,v in kwargs.items()]
    newnet = netchange(netlist, newparams)
    return newnet

def set_title(netlist, title):
    '''
    First line is always the title?
    Needs its own function because netinsert won't know how to identify and replace the title line
    '''
    return [title] + netlist[1:]

def flatten(nested):
    ''' flatten a tree of strings '''
    for i in nested:
            if isinstance(i, Iterable) and not isinstance(i, str):
                for subc in flatten(i):
                    yield subc
            else:
                yield i

### Spice directives
def param(name, value):
    return f'.PARAM {name}={value}'

def function(funcdef):
    funcdef = funcdef.replace(' ', '')
    return f'.FUNC {funcdef}'

def element(name, cathode, anode, val):
    '''
    For now we do not distinguish the element types.
    Spice will use the first letter of the name to decide what kind of element it is
    TODO: Overwrite value without specifying cathode and anode
    TODO: Autoname if name not given.
    Don't know how this would work, as we would need to be aware of the netlist that it is getting applied to, but we already return a string before that happens.
    '''
    return f'{name} {cathode} {anode} {val}'

# TODO: make functions for common elements.
#def resistor():
#def capacitor():
#def inductor():

def transient(start=0, stop=1, maxstep=1e-4, stopsteady=False):
    cmd = f'.tran 0 {stop} {start} {maxstep}'
    if stopsteady:
        cmd += ' steady'
    return cmd

def initial_condition(name, value):
    return f'.ic {name}={value}'

### Spice waveforms
# TODO add some useful functions that translate into these (e.g. triangle..)
def sine(freq, amp, offset=0, delay=0, phase=0, damping=0, ncycles=None):
    ''' Phase in degrees '''
    if ncycles is None:
        ncycles = ''
    return f'SINE({offset} {amp} {freq} {delay} {damping} {phase} {ncycles})'

def pulse(Voff, Von, trise, ton, tfall, period=None, delay=0, ncycles=None):
    if period is None:
        period = trise + ton + tfall + delay
    if ncycles is None:
        ncycles = ''
    return f'PULSE({Voff} {Von} {delay} {trise} {tfall} {ton} {period} {ncycles})'

def PWL(t, v):
    ''' Piecewise Linear waveform. Should probably use wav file instead if this gets long.'''
    interleaved = (val for pair in zip(t, v) for val in pair)
    interleaved_str = ' '.join(format(v) for v in interleaved)
    return f'PWL({interleaved_str})'

if 0:
    # Try changing all parameters by ±50%
    datalist = []
    for name, value in get_params(netlist).items():
        for v in np.linspace(0.5*value, 1.5*value, 5):
            data = runspice(netchange(netlist, param(name, v)))
            datalist.append(data)

    plt.figure()
    for data in datalist:
        plt.plot(data['time'], data['I(R1)'])

    #df = pd.DataFrame(datalist)


import json
import requests

subreddits = ['automation']

keywords = ['python', 'automation', 'scrape', 'automate']

headers = {"User-agent": "aws:com.sample:v0.1.4 by /u/someone"}

for sub in subreddits:
    url = 'https://www.reddit.com/r/' + sub + '/new.json'
    result = requests.get(url, headers = headers)
    parsed = json.loads(result.text)
#    print(json.dumps(parsed['data']['children'][0], indent=2))
    for post in parsed['data']['children']:
            if any(keyword in post['data']['title'] for keyword in keywords):
                print(post['data']['title'])


import sqlite3, datetime, json

class db_sql(object):

    def __init__(self):
        self.connection = sqlite3.connect('dados.db')
        self.c = self.connection.cursor()

    def execute(self, sql):
        self.c.execute(sql)
        return self.connection.commit()

    def return_data(self, sql):
        self.c.execute(sql)
        return self.c.fetchall()

text = {"ins":1,"receita":0,"nome":0,"interpol":0,"gerenc_log":0}

json_texto = json.dumps(text)

print(
    db_sql().execute(f'CREATE TABLE IF NOT EXISTS user_log(cpf text, nb text, cpf_rep text, res text, user text, date_time text);')
    #db_sql().execute(f'ALTER TABLE user_log ADD date_time text;')
    #db_sql().execute('ALTER TABLE users ADD last_login text;')
    #db_sql().return_data('SELECT * from users where user="gabrielyudenich"')
)


from flask import Flask, request
from flask_jwt_extended import JWTManager
import re
from flask_cors import CORS
from ducttapp.helpers import init_data_db

jwt = JWTManager()

def _after_request(response):
    allowed_origins = [
        re.compile('https?://(.*\.)?localhost'),
        # re.compile('https?://(.*\.)?teko\.vn'),
        # re.compile('https?://(.*\.)?phongvu\.vn')
    ]

    origin = request.headers.get('Origin')
    if origin:
        for allowed_origin in allowed_origins:
            if allowed_origin.match(origin):
                response.headers['Access-Control-Allow-Origin'] = origin

    response.headers['Access-Control-Allow-Credentials'] = 'true'
    response.headers['Access-Control-Allow-Methods'] = 'GET, POST, PUT, DELETE, OPTIONS'
    response.headers['Access-Control-Allow-Headers'] = 'Origin, X-Requested-With, Content-Type, Accept, Authorization'

    return response

def create_app():
    import config
    import os
    from . import api, models, extensions, services

    def load_app_config(app):
        app.config.from_object(config)
        app.config.from_pyfile('config.py', silent=True)

    app = Flask(
        __name__,
        instance_relative_config=True,
        instance_path=os.path.join(config.ROOT_DIR, 'instance')
    )

    load_app_config(app)
    app.after_request(_after_request)
    app.config['JWT_SECRET_KEY'] = config.FLASK_APP_SECRET_KEY
    app.config['JWT_TOKEN_LOCATION'] = ['cookies', 'query_string']
    app.config['JWT_COOKIE_CSRF_PROTECT'] = False
    app.config['JWT_BLACKLIST_ENABLED'] = True
    app.config['JWT_BLACKLIST_TOKEN_CHECKS'] = ['access', 'refresh']
    app.config['JWT_ERROR_MESSAGE_KEY'] = 'message'
    app.config['CORS_SUPPORTS_CREDENTIALS'] = True
    jwt.init_app(app)
    @jwt.token_in_blacklist_loader
    def check_if_token_in_blacklist(decrypted_token):
        jti = decrypted_token['jti']
        return models.RevokedToken.is_jti_blacklisted(jti=jti)

    api.init_app(app)
    models.init_app(app)
    services.init_app(app)
    CORS(app)
    return app

app = create_app()


# ------------------------------------------------------------------------------
# PBRT exporter - python  plugin for Maya by Vladimir (Volodymyr) Kazantsev
# 2013
#
# Based on a Python LuxMaya translation by Doug Hammond, in turn base on translation of the c++ luxmaya exporter,
# in turn based on original maya-pbrt c++ exporter by Mark Colbert
#
# This file is licensed under the GPL (the original exporter uses that license)
# http://www.gnu.org/licenses/gpl-3.0.txt
#
# $Id$
#
# ------------------------------------------------------------------------------
#
# This script loads the various commands and nodes to allow PBRT integration into Maya.
# This file is purely a loader, all the real classes are in the PBRT package.
#
# ------------------------------------------------------------------------------

import sys, os.path
from maya import OpenMaya
from maya import OpenMayaMPx

#GUI/Commands
from PBRT.Commands.pbrtbatch import pbrtbatch
from PBRT.Commands.gui import mainMenu

import pkgutil
import PBRT.Nodes

# ------------------------------------------------------------------------------

# Global list of commands that we want to register with Maya.
pbrtCommands = [
    pbrtbatch
]

# We build the list of nodes by scanning the Nodes directory
nodesPath = os.path.dirname(PBRT.Nodes.__file__)
pbrtNodes = [name for _, name, _ in pkgutil.iter_modules([nodesPath])]

pbrtRegisteredNode = []

# initialize the script plug-ins
def initializePlugin(mobject):
    """
    Start the plugin by registering all commands and nodes etc that this plugin provides
    """

    mplugin = OpenMayaMPx.MFnPlugin(mobject, "Vlad Volodymyr Kazantsev", "0.1", "Any")

    # there are multiple reload() calls in various places in code
    # Those are used to load updated code without restarting Maya
    #reload(PBRT.Commands.pbrtbatch)

    try:

        # Register commands
        for command in pbrtCommands:
            mplugin.registerCommand( command.commandName(), command.commandCreator )

        #Register nodes
        for nodeName in pbrtNodes:
            node = None
            try:
                __import__('PBRT.Nodes', fromlist=[nodeName])
                module = sys.modules['PBRT.Nodes.'+nodeName]
                reload(module)
                node = module.__dict__[nodeName]
            except ImportError:
                OpenMaya.MGlobal.displayError( "Failed to load "+nodeName )

            # abstract node classes should return None in nodeName() method
            isNode = getattr(node, 'nodeName', None)
            if isNode is None:
                continue

            pbrtRegisteredNode.append(node)
            mplugin.registerNode(
                node.nodeName(),
                node.nodeId(),
                node.nodeCreator,
                node.nodeInitializer,
                node.nodeType(),
                node.nodeClassify()

            )

            AETemplateName = "AE"+nodeName+"Template"
            try:
                __import__('PBRT.AETemplates', fromlist=[AETemplateName])
                module = sys.modules['PBRT.AETemplates.'+AETemplateName]
                reload(module)
            except ImportError:
                OpenMaya.MGlobal.displayError( "Failed to load "+AETemplateName )

        if OpenMaya.MGlobal.mayaState() == OpenMaya.MGlobal.kInteractive:
            # Create Top menu
            mainMenu().make()

        OpenMaya.MGlobal.displayInfo('PBRTMayaPy: Plugin loaded')

    except:
        OpenMaya.MGlobal.displayError( "Failed to register PBRT Plugin" )
        raise

# uninitialize the script plug-in
def uninitializePlugin(mobject):
    """
    Unregister all commands and nodes etc that the initializePlugin() method registered with Maya
    """

    mplugin = OpenMayaMPx.MFnPlugin(mobject)
    try:
        # deregister commands
        for command in pbrtCommands:
            mplugin.deregisterCommand ( command.commandName() )

        # deregister nodes
        for node in pbrtRegisteredNode:
            mplugin.deregisterNode( node.nodeId() )

        # OpenMaya.MGlobal.displayInfo("Plugin Removed OK.")
    except:
        OpenMaya.MGlobal.displayError( "Failed to deregister plugin" )
        raise


from typing import List
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
from datetime import datetime, timedelta

class DistScorer():
	'''
		Simple distance scorer, approximates track to be linear (valid because short).
		the prevlastpos attribute could be used to distiguish forward from backwards motion
	'''
	def __init__(self):
		self.prevlastpos 		: np.ndarray
		self.lastpos 			: np.ndarray
		self.last_cte 			: float
		self.distances			= []
		self.speeds				= []
		self.times  			= []
		self.total_frames 		= []
		self.scores 			= []
		self.current_race_dist  : float
		self.current_race_time 	: float
		self.start_time 		: datetime

	def first_point(self, info : dict):
		self.lastpos = np.array(info["pos"])

		self.prevlastpos = np.array(info["pos"])

		self.last_cte = info["cte"]

		self.current_race_dist = 0
		self.start_time = datetime.now()

	def add_point(self, info : dict):
		newpos = np.array(info["pos"])
		new_cte = info["cte"]
		cte_diff = abs(new_cte - self.last_cte)
		pos_diff = newpos - self.lastpos
		pos_diff_norm = np.linalg.norm(pos_diff)
		dist = np.sqrt(abs(pos_diff_norm ** 2 - cte_diff ** 2))
		if (np.linalg.norm(newpos - self.lastpos) > np.linalg.norm(newpos - self.prevlastpos)):
			dist = -1 * dist
		self.current_race_dist += dist
		self.last_cte = new_cte
		self.prevlastpos = self.lastpos
		self.lastpos = newpos

	def end_race(self, total_frames):
		self.current_race_time = (datetime.now() - self.start_time).total_seconds()

		self.distances.append(self.current_race_dist)

		self.total_frames.append(total_frames)

		self.times.append(self.current_race_time)

		self.scores.append(self.get_current_race_score())

		self.speeds.append(self.get_current_race_speed())

	def get_current_race_time(self):
		return (datetime.now() - self.start_time).total_seconds()

	def get_current_race_score(self):
		return (self.current_race_dist / np.sqrt(self.get_current_race_time()))

	def get_current_race_dist(self):
		return (self.current_race_dist)

	def get_current_race_speed(self):
		return (self.current_race_dist / self.get_current_race_time())

	def plot(self):
		plt.plot(self.total_frames, self.distances)
		plt.label("Scores per frame")
		plt.show()


import numpy as np
import cv2
import matplotlib.pyplot as plt

original_image = cv2.imread("../spikes/portal.jpg")

img = original_image

edges = cv2.Canny(img,150,200)
figure_size=15
plt.figure(figsize=(figure_size,figure_size))
plt.subplot(1,2,1),plt.imshow(img)
plt.title('Original image'), plt.xticks([]), plt.yticks([])
plt.subplot(1,2,2),plt.imshow(edges,cmap='gray')
plt.title("Edge Image"), plt.xticks([]), plt.yticks([])
plt.show()


from nltk.corpus import reuters
from nltk.stem.snowball import SnowballStemmer
from nltk.corpus import stopwords
from scipy import spatial
from nltk import word_tokenize, sent_tokenize
from numpy import linalg as la
from numpy import mean as mn
from scipy import spatial
from copy import copy,deepcopy
import networkx as nx
import os
import csv
from functools import reduce
#from stackoverflow import mycode

def prepare (text):
    #print("text: " + str(text))
    stemmer = SnowballStemmer("portuguese")
    text=text.lower()
    #print("text: " + str(text))
    collectedText =word_tokenize(text)
    #print("finaltext2: " + collectedText)
    tokensMinusPunctuation = [x.rstrip(',\'.&`>0123456789<!?;\")(') for x in collectedText]
    #print("tokensMinusPunctuation:" + str(tokensMinusPunctuation))
    tokensMinusStopwords = [word for word in tokensMinusPunctuation if word not in stopwords.words('portuguese')]
    tokensMinusEmpty = [word for word in tokensMinusStopwords if word!= ""]
    finalText = [stemmer.stem(word) for word in tokensMinusEmpty]
    return finalText

def generateGraph(text):
    dG = nx.DiGraph()
    for i, word in enumerate(text):
        try:
            nextWord = text[i+1]
            if not dG.has_node(word):
                dG.add_node(word)
                dG.node[word]['count'] = 1
            else:
                dG.node[word]['count'] += 1
            if not dG.has_node(nextWord):
                dG.add_node(nextWord)
                dG.node[nextWord]['count']=0
            if not dG.has_edge(word, nextWord):
                dG.add_edge(word, nextWord, weight=1)
            else:
                dG[word][nextWord]['weight']+=1
        except IndexError:
            if not dG.has_node(word):
                dG.add_node(word)
                dG.node[word]['count'] = 1
            else:
                dG.node[word]['count'] += 1
        except:
            raise
    return dG

def searchCycles(dg):
    cycles = list(nx.simple_cycles(dG))
    print("simple search finished. going for big one")
    sizes = [len(g) for g in cycles]
    c1 = len([g for g in sizes if g==1])
    c2 = len([g for g in sizes if g==2])
    c3 = len([g for g in sizes if g==3])
    weakComp = 0
    if len(sizes)>0:
        weakComp = max(sizes)
    try:
        sCycles = list(nx.find_cycle(dG, orientation='original'))
        strongComp =max([len(s) for s in sCycles])
    except:
        strongComp=0
    return c1,c2,c3,weakComp,strongComp

def avgDegree(dG):
    return float(len(dG.edges))/float(len(dG))

def prepareTwo (text):
    text=[f.lower() for f in text]
    final =[word_tokenize(word) for word in text]
    finalText = ""
    for a in final:
        finalText = finalText + str(a) + " "

    return finalText

def normalize(Originalvector):
    vector = Originalvector
    norm = la.norm(vector)
    for i in range(len(vector)):
        if norm != 0:
            vector[i] = vector[i] / norm
        else: return vector
        return vector

def vectorSum(vectorA, vectorB):
    if (type(vectorA) is list) & (type(vectorB) is list):
        result = [vectorA[0]+vectorB[0],vectorA[1]+vectorB[1],vectorA[2]+vectorB[2],vectorA[3]+vectorB[3],
              vectorA[4]+vectorB[4],vectorA[5]+vectorB[5],vectorA[6]+vectorB[6],vectorA[7]+vectorB[7]]
    elif type(vectorA) is list:
        result = vectorA
    elif type(vectorB) is list:
        result = vectorB
    else:
        result = [0]*8
    return result

def vectorDistanceNormalized(vectorA, vectorB):
    return spatial.distance.euclidean(vectorA, vectorB)*spatial.distance.cosine(vectorA, vectorB)
#The thing here is that just cossine isn't enough, so the metric we are using is the cossine TIMES the distance between vectors. Hopefully this will be enough.

def vectorCoolCosine(vectorA, vectorB):
    return vectorDistanceNormalized(vectorA, vectorB)

def fixEntryCorpus(row): #Pra ficar mais fácil de tratar, essa função transforma os 8 ultimos caras da entrada num unico vetor8
    result = [row[0]]
    result.append([float(row[1]),float(row[2]),float(row[3]),float(row[4]),float(row[5]),float(row[6]),float(row[7]),float(row[8])])
    return result

def checkCorpus(word):
    for row in emotionCorpus:
        if word == row[0]:
            result = row[1].copy
            return result
        return [0,0,0,0,0,0,0,0]

def calculateMeanLine(lineVector):
    return mn(lineVector)

if __name__ == '__main__':
    path = "control/"
    sentences = []
    output = open('control_data.txt', 'w')
    for filename in os.listdir(path):
        content = open(path+filename,encoding="latin1")
        lines = content.readlines()
        print(filename)
        strLines = str(lines)
        paragraphs = strLines.split("\n")

        #sentences.append(lines)
        content.close()

    #print(sentences[:], sep=' ')
        for sent in paragraphs:
            phrases =sent.split(".")
            filterEmpty = [line for line in phrases if len(line)>0]

            for text in filterEmpty:
                #print("text"+text)
                text2=prepare(text)
                print(filename)
                if len(text2)>0:
                    dG = generateGraph(text2)
                    print(len(dG))
                    repeatedEdges=0
                    for edge in dG.edges():
                        repeatedEdges+=dG[edge[0]][edge[1]]['weight']-1

                    print("search started")
                    c1,c2,c3,maxCycle,maxStrongCycle = searchCycles(dG)
                    print("search finished.")
                    avg = avgDegree(dG)
                    ##daqui comeca a analise de sentimentos
                    with open("NRCEmotion.csv", newline='') as csvfile:
                        emotionCorpus = csv.reader(csvfile, delimiter=';', dialect='excel',)
                        emotionCorpus = map(fixEntryCorpus, emotionCorpus)
                        #corpus = open("corpus.txt")
                        tokenList = phrases[:]
                        line = []
                        sentences = []
                        counter = 0
                        sentenceVector = [0,0,0,0,0,0,0,0]
                        emotionalVariance = []
                        for sentence in tokenList:
                            for word in sentence:
                                line.append(checkCorpus(word))
                            for vector in line:
                                sentenceVector = vectorSum(vector, sentenceVector)
                            sentences.append(sentenceVector)
                            line = []
                            sentenceVector = [0,0,0,0,0,0,0,0]
                        for i in range(0,len(sentences)):
                            try:
                                emotionalVariance.append(vectorCoolCosine(sentences[i],sentences[i+1]))
                                i += 1
                            except IndexError:
                                break
                        ##aqui termina
                        adding = reduce ((lambda x,y: x+y),emotionalVariance)
                        adding = adding/len(emotionalVariance)
                        components =[repeatedEdges,c1,c2,c3,maxCycle,maxStrongCycle,avg]
                        components.append(adding)
                        components.append(1)
                        components.append(0)
                        components.append(0)
                        if (repeatedEdges+c1+c2+c3+maxCycle+maxStrongCycle)>0:
                            answer= "["
                            for i in components:
                                answer = answer + str(i) + ","
                            output.write(answer[:-1]+"]\n")
                            print(answer)
    output.close()