import numpy as np

from scipy.signal.windows import triang, blackmanharris
from peakinterp import peakinterp
from genspecsines import genspecsines
from f0detectiontwm import f0detectiontwm
from f0detectionyin_gabi import f0detectionyin
from custom_windows import custom_triang, custom_blackmanharris

def harmonicmodel(
    x: np.ndarray,
    fs: float,
    w: np.ndarray,
    N: int,
    t: int,
    nH: int,
    minf0: float,
    maxf0: float,
    f0et: float,
    maxhd: float,
    f0detection: str = 'twm',
    Ns: int = 1024,
    H: int = 256
):
    """
    Analysis and synthesis of a audio signal using the sinusoidal harmonic model

    Args:
        x (np.ndarray): Input signal
        fs (float): Sampling frequency
        w (np.ndarray): Analysis window
        N (int): FFT size for analysis
        t (float): Threshold in negative dB
        nH (int): Maximum number of harmonics
        minf0 (float): Minimum f0 in the signal
        maxf0 (float): Maximum f0 in the signal
        f0et (float): Error threshold in the f0 detection
        maxhd (float): Maximum relative deviation in harmonic detection
        f0detection (str, optional): Method for f0 detection. Accepts 'yin' and 'twm'. Defaults to 'yin'.
        Ns (int, optional): FFT size for synthesis
        H (int, optional): Hop size for analysis and synthesis

    Returns:
        y (np.ndarray): Output signal
    """
    # tratamento de entradas
    if f0detection not in ['twm', 'yin']:
        raise Exception("Método inválido de detecção de f0. Escolha \'yin\' ou \'twm\'")

    M = len(w)                                                                                # analysis window size
    N2 = N//2 + 1                                                                               # size of positive spectrum
    audio_len = len(x)                                                                          # length of input audio signal array
    hNs = Ns//2                                                                                # half synthesis window size
    # hM matlab: 511.5
    # hM python: 511
    hM = (M - 1)//2                                                                             # half analysis window size
    y = np.zeros(audio_len + Ns//2)	                                                            # initialize output array
    w = w / np.sum(w)                                                                           # normalize analysis window

    sw = np.zeros(Ns)

    # como o comportamento da janela triangular estava diferente,
    # fiz uma funcao propria seguindo a regra da funcao do matlab
    # para manter o mesmo comportamento
    ow = custom_triang(2*H - 1)
    # ow = triang(2*H - 1, sym=False)                                                           # overlapping window

    # slidado em -1 em relacao ao matlab
    ovidx = np.arange((Ns//2 + 1 - H), (Ns//2 + H), dtype=np.int64)                             # overlap indexes
    sw[ovidx] = ow[:(2*H)]

    # como o comportamento da blackmanharris estava diferente,
    # fiz uma funcao propria seguindo a regra da funcao do matlab
    # para manter o mesmo comportamento da blackmanharris periodica
    # bh = custom_blackmanharris(Ns)
    bh = blackmanharris(Ns, sym=False)                                                        # synthesis window
    bh = bh / np.sum(bh)                                                                      # normalize synthesis window

    sw[ovidx] = sw[ovidx] / bh[ovidx]

    # pin matlab: 512.5
    # pin python: 512
    pin = np.maximum(H + 1, 1 + hM)                                                             # initialize audio signal pointer to middle of analysis window
    pend = audio_len - hM                                                                       # last sample to start a frame

    while pin < pend:

        # analysis
        fftbuffer = np.zeros(N)                                                                 # reset buffer for FFT
        xw = x[(pin - hM - 1):(pin + hM + 1)] * w[:M]

        fftbuffer[:((M + 1)//2)] = xw[((M + 1)//2):M]                                           # zero-phase window in fftbuffer
        fftbuffer[(N - (M - 1)//2):N] = xw[:((M - 1)//2)]

        X = np.fft.fft(fftbuffer)                                                               # compute the FFT
        mX = 20 * np.log10(np.abs(X[:N2]))                                                      # magnitude spectrum of positive frequencies
        pX = np.unwrap(np.angle(X[:N2]))                                                        # unwrapped phase spectrum

        ploc = np.nonzero(
            (mX[1:N2-1] > t) * \
            (mX[1:N2-1] > mX[2:N2]) * \
            (mX[1:N2-1] > mX[:N2-2])
        )[0] + 1                                                                                # find peaks
        ploc, pmag, pphase = peakinterp(mX, pX, ploc)                                           # refine peak values

        if f0detection == 'twm':
            f0 = f0detectiontwm(mX, fs, ploc, pmag, f0et, minf0, maxf0)
        # callyin.py
        if f0detection == 'yin':
            yinws = round(0.015 * fs) + round(0.015 * fs) % 2                                   # using approx. a 15 ms window for yin and making it even
            yb = pin - yinws//2
            ye = pin + yinws + yinws//2
            if (yb < 1) or (ye > len(x)):
                f0 = 0
            else:
                f0 = f0detectionyin(x[(yb-1):ye], fs, yinws, minf0, maxf0)                      # find f0

        hloc = np.zeros(nH)                                                                     # initialize harmonic locations
        hmag = np.zeros(nH) - 100                                                               # initialize harmonic magnitudes
        hphase = np.zeros(nH)                                                                   # initialize harmonic phases
        hf = (f0 > 0) * f0 * np.arange(1, nH + 1)                                               # initialize harmonic frequencies
        hi = 0                                                                                # initialize harmonic index

        npeaks = len(ploc)
        while (f0 > 0) and (hi < nH) and (hf[hi] < fs/2):
            peak = np.abs(ploc[:npeaks]/N*fs - hf[hi])                                          # find harmonic peaks
            dev = np.min(peak)                                                                  # closest peak
            pei = np.argmin(peak)

            if ((hi == 0) or not (hloc[:hi] == ploc[pei]).any()) and (dev < maxhd * hf[hi]):
                hloc[hi] = ploc[pei]                                                            # harmonic locations
                hmag[hi] = pmag[pei]                                                            # harmonic magnitudes
                hphase[hi] = pphase[pei]                                                        # harmonic phases
            hi += 1                                                                             # increase harmonic index

        hloc[:hi] = (hloc[:hi] != 0) * ((hloc[:hi] - 1)*Ns/N + 1)                                  # synthesis locs

        # synthesis
        Yh = genspecsines(hloc[:hi], hmag, hphase, Ns)                                      # generate spec sines
        yh = np.fft.fftshift(np.real(np.fft.ifft(Yh)))                                          # time domain of sinusoids
        y[(pin - hNs):(pin + hNs)] = y[(pin - hNs):(pin + hNs)] + sw * yh[:Ns]                  # overlap-add

        pin = pin + H                                                                           # advance the audio signal pointer

    return y


# coding:utf-8

import requests

# proxies = {"http":"http://101.96.11.39:8080"}
proxies = {"http":"http://101.236.35.98:8866"}
headers = {"User-Agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1"}
reponse = requests.get("http://www.baidu.com",proxies=proxies,headers=headers)
# print(reponse.content.decode())
print(reponse.status_code)


# 템플릿과 데이터를 이용해 HTML 파일을 생성하는 render 포함
from django.shortcuts import render
from django.views import generic

# Create your views here.
# 뷰는 HTTP 요청을 처리하고, 데이터베이스에서 데이터를 가져오고,
# HTML 템플릿으로 데이터를 랜더링하고 생성된 HTML을 HTTP응답으로 반환하여 사용자들에게 보여주는 함수이다.
# (index 뷰는 이 구조(모델)을 따라간다
# index는 데이터베이스 안의 모델들의 레코드 개수 및 정보를 가져오고 그 정보를 템플릿으로 전달한다

from catalog.models import Book, Author, BookInstance, Genre
# 우리의 모든 뷰들 안에서 데이터에 접근하는데 필요한 모델 클래스 import

def index(request) :
    """사이트 홈페이지 보기 기능"""

    # 일부 주요 개체의 개수 생성
    num_books = Book.objects.all().count()
    num_instances = BookInstance.objects.all().count()
    # 모델 클래스들에서 레코드들의 개수를 가져옴

    # 사용 가능한 책(상태)
    num_instances_available = BookInstance.objects.filter(status__exact='a').count()
    # 상태 필드에서 'a'값을 가지는 BookInstance 객체들 목록도 가져옴

    # 'all()'은 기본적으로 내포되어 있다
    num_authors = Author.objects.count()

    context = {
        'num_books' : num_books,
        'num_instances' : num_instances,
        'num_instances_available' : num_instances_available,
        'num_authors' : num_authors,
    }

    # 컨텍스터 변수의 데이터를 사용하여 HTML 템플릿 index.html을 렌더링한다.
    return render(request, 'index.html', context=context)
    # render() : HTML 페이지 생성 및 페이지를 응답으로 반환하는 함수
    # render()의 매개변수들
    # request : HttpRequest인 원본 객체
    # 데이터에 대한 플레이스홀더들을 갖고 있는 HTML 템플릿
    # context

# books/ 링크로 들어오면 실행되는 모델
class BookListView(generic.ListView) :
    model = Book

    def get_queryset(self): # 정쟁이 포함된 책 5권 얻기
        return Book.objects.filter(title__icontains='war')[:5]

    def get_context_data(self, **kwargs):
        # 컨텍스트를 얻으려 기본 구현을 호출
        context = super(BookListView, self).get_context_data(**kwargs)
        # 데이터를 생성하고 컨텍스트에 추가
        context['some_data'] = 'This is just some data'
        return context


"""
The below program get/input data from a excel sheet and will write/print the unique data on text document.
"""
import openpyxl as op
import os

drive = "" #! Please enter the drive path.
geo_input_file = "" #! Please enter the input file name.
geo_output_file = "" #! Please enter the output file name.

geo_input_path = drive + geo_input_file
geo_output_path = drive + geo_output_file

# * Loading all the district from geolocation excelsheet.

geolocation_city = []
wb1 = op.load_workbook(geo_input_path)
ws1 = wb1.get_sheet_by_name("Sheet1")
rm1 = ws1.max_row
for loc in range(2, (rm1 + 1)):
    geo_cit = ws1.cell(row = loc, column = 1).value
    if geo_cit not in geolocation_city:
        geolocation_city.append(geo_cit)
        output = open(geo_output_path, 'a')
        output.write(geo_cit + "\n")
        output.close()
print("All Done")

excelWork/geolocation city import and export.py


# coding:utf8
import tornado.ioloop
import tornado.web
import re
import os

import logging
logging.basicConfig(level = logging.INFO,format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)
logger.setLevel(level = logging.INFO)
handler = logging.FileHandler("log.txt")
handler.setLevel(logging.INFO)
logger.addHandler(handler)

from dotenv import load_dotenv
load_dotenv(verbose=True)

from knockd_core import LightHouse

# --LOAD--ENV--
MESSAGE_SUCCESS = os.getenv('MESSAGE_SUCCESS')
MESSAGE_FAILURE = os.getenv('MESSAGE_FAILURE')

KNOCK_REFERER = os.getenv('KNOCK_REFERER')

FW_PERMIT_PORTS = os.getenv('FW_PERMIT_PORTS')
LIGHTHOUSE_INSTANCE_IDS = os.getenv('LIGHTHOUSE_INSTANCE_IDS')
KNOCK_REQUEST_PATH = os.getenv('KNOCK_REQUEST_PATH')

class MainHandler(tornado.web.RequestHandler):
    def get(self):
        self.set_status(500)
        self.write(MESSAGE_FAILURE)

class KnockdoorHandler(tornado.web.RequestHandler):
    def get(self):
        # 当前服务器看到的真实远程IP地址，默认方式
        knocking_ip = self.request.remote_ip
        method_to_get_client_ip  = os.getenv('METHOD_TO_GET_CLIENT_IP', "1")
        if "3" == method_to_get_client_ip:
            # 客户端主动携带的IP地址，不安全，容易被篡改
            knocking_ip = self.request.headers.get('ip', '127.0.0.1')
        elif "2" == method_to_get_client_ip:
            # 支持Cloudflared代理的原始IP地址，但CF看到的IP地址可能不是真实IP地址
            # CN网内各种奇葩的广电、华数字的路由和NAT策略会让你疯掉
            knocking_ip = self.request.headers.get("X-Real-IP") or \
                self.request.headers.get("X-Forwarded-For") or \
                    self.request.remote_ip

        logging.info(f"Knocking from client:{knocking_ip}")
        self.set_status(500)
        if re.match(r"\d+\.\d+\.\d+\.\d+", knocking_ip):
            if self.knock_lighthouse(knocking_ip):
                self.write(MESSAGE_SUCCESS)
                return
        self.write(MESSAGE_FAILURE)

    def knock_lighthouse(self, knocking_ip='127.0.0.1'):
        """
        敲门lighthouse
        """
        client = self.request.headers.get('Location', 'DEFAULT')
        referer = self.request.headers.get('Referer', '-')
        if referer == KNOCK_REFERER:
            desc = "Knockd-" +  client
            instance_ids = os.getenv('TENCENT_CLOUD_LIGHTHOUSE_INSTANCE_IDS').strip().strip(',').strip().split(',')
            secret_id = os.getenv('TENCENT_CLOUD_SECRET_ID')
            secret_key = os.getenv('TENCENT_CLOUD_SECRET_KEY')
            # 每一个轻量级主机实例，单独处理
            for instance_id in instance_ids:
                if instance_id == '':continue
                tcp_ports = os.getenv('FW_PERMIT_PORTS_TCP').strip().strip(',').strip()
                tcp_ports = ','.join(tcp_ports.split(','))
                udp_ports = os.getenv('FW_PERMIT_PORTS_UDP').strip().strip(',').strip()
                udp_ports = ','.join(udp_ports.split(','))
                # 先处理TCP，后处理UDP
                if tcp_ports != '' and tcp_ports != ',':
                    LightHouse(instance_id, secret_id, secret_key).add_knock_ip(knocking_ip, tcp_ports, 'TCP', descrption=desc)
                if udp_ports != '' and udp_ports != ',':
                    LightHouse(instance_id, secret_id, secret_key).add_knock_ip(knocking_ip, udp_ports, 'UDP', descrption=desc)
            return True
        return False

"""A Knockd web service"""
def knockdoor_app():
    return tornado.web.Application([
        (r"/", MainHandler),
        (KNOCK_REQUEST_PATH, KnockdoorHandler)
    ])

if __name__ == '__main__':
    app = knockdoor_app()
    int_bind_port = int(os.getenv('BIND_PORT', "8080"))
    str_bind_ip = os.getenv('BIND_IP', "0.0.0.0")
    app.listen(int_bind_port, str_bind_ip)
    tornado.ioloop.IOLoop.current().start()


import numpy as np
from random import randint
from time import time

import torch
import torch.nn as nn
import torch.nn.functional as F  # this includes tensor functions that we can use in backwards pass
import torchvision
import torchvision.datasets as datasets
from torchvision import transforms
import torch.optim as optim

from Trainer import Trainer

# this function does a depth-first search across a matrix of 0s and 1s,
# looking for a path of 1s between the first and last columns of the matrix
# such a path is called a 'percolation path'

def percolate(mat):
    """
    Returns True if there is a percolation path of 1s from col 0 col -1 of a matrix of 0s and 1s
    """
    nrows = mat.shape[0]
    ncols = mat.shape[1]
    frontier = set()
    for i in range(0, nrows):
        if mat[i, 0]:
            frontier.add((i, 0))
    explored = set()
    flag = False  # this will be returned if the frontier becomes empty without finding a
    # filled pixel in the right-most column
    while frontier:  # frontier evaluates to True in this context if it is non-empty
        r, c = frontier.pop()
        explored.add((r, c))
        if r > 0:  # North
            if mat[r - 1, c]:
                coords = (r - 1, c)
                if coords not in explored:
                    if coords not in frontier:  # this order of testing is necessary since each element of explored has been in frontier
                        frontier.add(coords)
        if c < ncols - 1:  # East
            if mat[r, c + 1]:
                if c + 1 == mat.shape[1] - 1:  #  Hurray, we have percolated to the last column
                    flag = True
                    break
                coords = (r, c + 1)
                if coords not in explored:
                    if coords not in frontier:  # this order of testing is necessary since each element of explored has been in frontier
                        frontier.add(coords)
        if r < nrows - 1:  # South
            if mat[r + 1, c]:
                coords = (r + 1, c)
                if coords not in explored:
                    if coords not in frontier:  # this order of testing is necessary since each element of explored has been in frontier
                        frontier.add(coords)
        if c > 0:  # West
            if mat[r, c - 1]:
                coords = (r, c - 1)
                if coords not in explored:
                    if coords not in frontier:  # this order of testing is necessary since each element of explored has been in frontier
                        frontier.add(coords)
    return flag

def make_percolation_dataset(side=8, threshold=0.42, n_examples=10):
    """
    This function generates an array of random images, in the form needed for Keras, and
    then labels them as percolati# ng or not, using the percolate function. Roughly 50% of the
    images will have class 1 (percolating), so the dataset is likely to be reasonably balanced.
    """
    X_data = (np.random.random([n_examples, side, side, 1]) > threshold).astype(float)
    Y_data = np.zeros([n_examples, 1])
    for i in range(0, n_examples):
        if percolate(X_data[i, :, :, 0]):
            Y_data[i, 0] = 1
    dataset = [{'image': torch.tensor(x.astype(np.float32)).reshape(1, side, side),
                'label': torch.tensor(float(y)).reshape(1)} for x, y in zip(X_data, Y_data)]
    print(sum([x['label'].item() for x in dataset]) / len(dataset))
    return dataset

# this function calculates the loss and error rate on the validation set
# assuming that the output of the neural net nn is a single prediction probability, from a single sigmoid neuron.

def accuracy_and_loss(net, dataloader):
    loss_function = nn.BCELoss()
    total_correct = 0
    total_loss = 0.0
    total_examples = 0
    n_batches = 0
    with torch.no_grad():  # we do not neet to compute the gradients when making predictions on the validation set
        for data in dataloader:
            images, labels = data
            outputs = net(images)
            batch_loss = loss_function(outputs, labels)  # this is averaged over the batch
            n_batches += 1
            total_loss += batch_loss.item()
            total_correct += sum(
                (outputs > 0.5) == (labels > 0.5)).item()  # number correct in the minibatch
            total_examples += labels.size(
                0)  # the number of labels, which is just the size of the minibatch

    accuracy = total_correct / total_examples
    mean_loss = total_loss / n_batches

    return (accuracy, mean_loss)  # print( "Accuracy on test set: %d %%" %(100 * correct/total_examples))

# a nn with one hidden layer
class NN_one_hidden(nn.Module):

    def __init__(self):
        super(NN_one_hidden, self).__init__()
        self.layers = nn.Sequential(nn.Flatten(), nn.Linear(64, 100),  # try 100 hidden neurons
            nn.ReLU(),
            # note that we need the brackets after nn.ReLU() and nn.Sigmoid(), because these
            # are classes that need to be instantiated
            nn.Linear(100, 1),
            nn.Sigmoid())  # we are predicting only two classes, so we can use one sigmoid neuron as output

    def forward(self, x):  # computes the forward pass ... this one is particularly simple
        x = self.layers(x)
        return x

    def __getattr__(self, item):
        if item == 'device':
            return 'cpu'
        return super().__getattr__(item)

class NN_two_hidden(nn.Module):

    def __init__(self):
        super(NN_two_hidden, self).__init__()
        self.layers = nn.Sequential(nn.Flatten(), nn.Linear(64, 100),  # try 100 hidden neurons
            nn.ReLU(),
            # note that we need the brackets after nn.ReLU() and nn.Sigmoid(), because these
            # are classes that need to be instantiated
            nn.Linear(100, 100), nn.ReLU(), nn.Linear(100, 1),
            nn.Sigmoid())  # we are predicting only two classes, so we can use one sigmoid neuron as output

    def forward(self, x):  # computes the forward pass ... this one is particularly simple
        x = self.layers(x)
        return x

    def __getattr__(self, item):
        if item == 'device':
            return 'cpu'
        return super().__getattr__(item)

class PercolationTrainer(Trainer):
    def __init__(self, train_dl, dev_dl, test_dl, load_model=None, n_epochs=10):
        model = NN_one_hidden()
        if load_model is not None:
            model.load_state_dict(load_model.state_dict())
        optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
        super().__init__(model, train_dl=train_dl, dev_dl=dev_dl, test_dl=test_dl,
                         num_epochs=n_epochs, optimizer=optimizer, loss_function=torch.nn.BCELoss())

    def predict_batch(self, batch):
        images = batch['image']
        labels = batch['label']
        if len(labels.shape) > 1:
            labels = labels.squeeze(1)
        outputs = self.model(images).squeeze(1)
        return outputs, labels

    @staticmethod
    def performance(pred, gold):
        return ((pred > 0.5) == (gold > 0.5)).mean()

def define_and_train(NN_class, args, n_epochs, training_set, test_set, batch_size=32,
                     weight_decay=0.0, continue_net=None, pr=False):
    loss_function = nn.BCELoss()
    trainloader = torch.utils.data.DataLoader(training_set, batch_size=batch_size, shuffle=True)
    testloader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, shuffle=True)

    if continue_net == None:
        thenet = NN_class(**args)
    else:
        thenet = continue_net
    optimizer1 = optim.Adam(thenet.parameters(), weight_decay=weight_decay, lr=0.001)
    # optimizer1 = optim.SGD( thenet.parameters(), weight_decay=weight_decay , lr = 0.01,
    # momentum = 0.9)

    train_acc = []
    val_acc = []
    train_loss = []
    val_loss = []
    conv3x3_weight_grads = []
    conv1x1_weight_grads = []
    bias_grads = []

    start_time = time()

    for epoch in range(n_epochs):  # number of times to loop over the dataset

        total_loss = 0
        total_correct = 0
        total_examples = 0
        n_mini_batches = 0
        conv3x3_weight_grad = torch.zeros(3, 3)

        errors = []

        thenet.train()
        for i, mini_batch in enumerate(trainloader, 0):
            images, labels = mini_batch

            # zero the parameter gradients
            # all the parameters that are being updated are in the optimizer,
            # so if we zero the gradients of all the tensors in the optimizer,
            # that is the safest way to zero all the gradients
            optimizer1.zero_grad()

            outputs = thenet(images)  # this is the forward pass

            loss = loss_function(outputs, labels)

            loss.backward()  # does the backward pass and computes all gradients

            # conv3x3_weight_grad += thenet.conv3x3.weight.grad.data[0,0]

            optimizer1.step()  # does one optimisation step

            n_mini_batches += 1  # keep track of number of minibatches, and collect the loss for each minibatch
            total_loss += loss.item()  # remember that the loss is a zero-order tensor
            # so that to extract its value, we use .item(), as we cannot index as there are no dimensions

            # keep track of number of examples, and collect number correct in each minibatch
            total_correct += sum((outputs > 0.5) == (labels > 0.5)).item()
            total_examples += len(labels)

            for j in range(len(labels)):
                if len(errors) < 10 and (outputs[j] > 0.5) != (labels[j] > 0.5):
                    errors.append(images[j])

        # calculate statistics for each epoch and print them.
        # You can alter this code to accumulate these statistics into lists/vectors and plot them
        epoch_training_accuracy = total_correct / total_examples
        epoch_training_loss = total_loss / n_mini_batches
        # conv3x3_weight_grad /= n_mini_batches

        thenet.eval()
        epoch_val_accuracy, epoch_val_loss = accuracy_and_loss(thenet, testloader)
        duration = int(time() - start_time)

        print('Epoch %d loss: %.3f acc: %.3f val_loss: %.3f val_acc: %.3f time: %d:%d%d' % (
        epoch + 1, epoch_training_loss, epoch_training_accuracy, epoch_val_loss, epoch_val_accuracy,
        duration // 60, duration % 60 // 10, duration % 60 % 10))

        # if (epoch + 1) % 10 == 0:
        # print(thenet.conv3x3.weight)
        # print(thenet.conv3x3.bias)
        # print(thenet.conv1x1.weight)

        if pr:
            plt.imshow(errors[0].numpy().squeeze())
            plt.show()

        train_loss.append(epoch_training_loss)
        train_acc.append(epoch_training_accuracy)
        val_loss.append(epoch_val_loss)
        val_acc.append(epoch_val_accuracy)
        conv3x3_weight_grads.append(conv3x3_weight_grad)

    history = {'train_loss': train_loss, 'train_acc': train_acc, 'val_loss': val_loss,
               'val_acc': val_acc, 'conv3x3_weight_grads': conv3x3_weight_grads,
               'conv1x1_weight_grads': conv1x1_weight_grads, 'bias_grads': bias_grads}
    return (history, thenet)


import random
import time

import twython

from gonotego.common import events
from gonotego.settings import settings

def _tweet(client, text, tweet_id=None):
  # Assumes text is < 280 characters.
  # If tweet_id is None, tweets a non-reply tweet.
  # If tweet_id is not None, tweets a reply tweet.
  return client.update_status(
      status=text, in_reply_to_status_id=tweet_id)

def _tweet_thread(client, texts, tweet_id=None):
  # Assume each text is < 280 characters.
  # If tweet_id is None, tweets a non-reply tweet thread.
  # If tweet_id is not None, tweets a reply tweet thread.
  tweets = []
  for text in texts:
    if not tweets:
      tweet = _tweet(client, text, tweet_id=tweet_id)
    else:
      time.sleep(random.random())
      tweet = _tweet(
          client, text, tweet_id=tweets[-1]['id_str'])
    tweets.append(tweet)
  return tweets

def split_to_tweets(text, delimiter=None, limit=280):
  texts = []
  if delimiter and delimiter in text:
    # Presence of delimiter forces splitting into a new tweet.
    for t in text.split(delimiter):
      texts.extend(split_to_tweets(t, delimiter=None, limit=limit))
    return texts
  spacers = ['\n\n\n', '\n\n', '\n', ' ']
  preferred_breaks = ['.', '!', '?', ';', ',', '-', '+', '*', '&', ')']
  while len(text) > limit:
    for spacer in spacers + preferred_breaks:
      try:
        index = text[:limit].rindex(spacer)
        end_index = index
        start_index = index + len(spacer)
        if spacer in preferred_breaks:
          end_index += len(spacer)
        texts.append(text[:end_index])
        text = text[start_index:]
        break
      except ValueError:
        pass
    else:
      # None of the spacers matched.
      texts.append(text[:limit])
      text = text[limit:]

  if text:
    texts.append(text)
  return texts

def send(client, text, tweet_id=None):
  texts = split_to_tweets(text)
  return _tweet_thread(client, texts, tweet_id)

def get_oauth_tokens():
  user_id = settings.get('twitter.user_id')
  oauth_token = settings.get(f'twitter.user_ids.{user_id}.oauth_token')
  oauth_token_secret = settings.get(f'twitter.user_ids.{user_id}.oauth_token_secret')

  # TODO(dbieber): Add fallback to settings:
  # settings.get('TWITTER_ACCESS_TOKEN')
  # settings.get('TWITTER_ACCESS_TOKEN_SECRET')

  return oauth_token, oauth_token_secret

class Uploader:

  def __init__(self):
    self._client = None
    self.last_tweet_id = None

  @property
  def client(self):
    if self._client:
      return self._client
    oauth_token, oauth_token_secret = get_oauth_tokens()
    self._client = twython.Twython(
        settings.get('TWITTER_API_KEY'),
        settings.get('TWITTER_API_SECRET'),
        oauth_token,
        oauth_token_secret)
    return self._client

  def upload(self, note_events):
    client = self.client
    for note_event in note_events:
      if note_event.action == events.SUBMIT:
        text = note_event.text.strip()
        tweets = send(client, text, self.last_tweet_id)
        self.last_tweet_id = tweets[-1]['id_str']
      elif note_event.action == events.END_SESSION:
        self.end_session()

  def handle_inactivity(self):
    self._client = None
    self.end_session()

  def handle_disconnect(self):
    self._client = None
    self.end_session()

  def end_session(self):
    self.last_tweet_id = None


import warnings
warnings.filterwarnings("ignore")
import sys
sys.tracebacklimit=0
import os
import numpy as np
import random
import re

lim=600 #this is a placeholder integer that decides the number of input to be train
#We first read the file data
path = os.path.dirname(__file__)
file = os.path.join(path, 'wdbc.data')
file_object = open(file, 'r')

#Split dataset into separate points (as strings)
string_points = file_object.read().split('\n')
string_points.pop(-1)
random.shuffle(string_points) #Randomize (avoid bias)
num = 30 #number of appropriate features that we want

#Initialize dataset class arrays
point_array = np.empty((0,num))
y_labels = np.empty(0) #creating a array using the numPy library

#Some pre-processing of the data is required
#Trim data points
#Format as np.arrays
#Add to class arrays (Benign or Malignant)
for point in string_points:
    #print(point)
    point = point.split(',')
    #print(y_labels)
    if 'M' in point: #if malignant
        y_labels = np.append(y_labels, [1], axis=0)
    else: #if benign (can only be labeled 'B' or 'M')
        y_labels = np.append(y_labels, [0], axis=0)

    point = point[2:] #trim for only the important features
    temp = np.array(point) #convert to numpy array
    temp = temp.astype(float) #cast as float array
    point_array = np.append(point_array, [temp], axis=0)

#Split training and testing data
experience_matrix = point_array[0:500] #the first 500 hundred entries are the experience set
experience_matrix_y = y_labels[0:500]
test_matrix = point_array[500:] #the last 300 are the test entries
test_matrix_y = y_labels[500:]

#We now use Sklearn linear regression to go about this
from sklearn.linear_model import LogisticRegression
# all parameters not specified are set to their defaults
logisticRegr = LogisticRegression(random_state=0)
logisticRegr.fit(experience_matrix, experience_matrix_y)

Y_pred = logisticRegr.predict(test_matrix)

#We now give the user a choice, if he/she/... want to enter their own data or they want to test the model through the test data
print("Hello user. You have two choices. Feed in your own data or just test the accuracy.")
print("To enter your own data press 1 and then enter")
print("To just test accuracy and look at the test data set press 2 and then enter")
o=input("Put in your choice\n")
# The first choice is the user/doctor
if(o=='1'):
    option=input("Enter the file name that contains the data to be run\n")
    path = os.path.dirname(__file__)
    file = os.path.join(path, option)
    file_object = open(file, 'r')
    ex_matrix_y=np.empty(0)
    final_y=np.empty(0)
    #Split dataset into separate points (as strings)
    spoints = file_object.read().split('\n')
    ex_matrix_y=spoints[0].split(',')
    for i in range(len(ex_matrix_y)):
        t = float(ex_matrix_y[i])
        final_y=np.append(final_y,[t], axis=0)

    from sklearn.metrics import confusion_matrix
    cm = confusion_matrix(test_matrix_y, Y_pred)
    y_f=[final_y]
    y=logisticRegr.predict(y_f)

    if(y[0]==1):
        print("The tumor is Malignant. You should consult a doctor")
    else:
        print("The tumor is Benign. You are ~90% safe. Still consult a doctor if you feel like it.")

elif(o=='2'):
    #print(final_y)
    y_pred_l=len(Y_pred)
    count=0
    for i in range(y_pred_l):
        if Y_pred[i]==test_matrix_y[i]:
            count+=1

    accuracy=count/y_pred_l
    print("The test result matrix is as follows")
    print("1 means Malignant and 0 means Benign")
    print(test_matrix_y)
    print("The accuracy in which this bot does the classification is")
    accuracy=round(accuracy*100,2)
    print((str)(accuracy)+"%")

else:
    print("Please enter only 1 or 2. Now rerun me if you want to test me")


from io import BytesIO
from PIL import Image as Img
import cv2 as cv

import matplotlib.image as mpimg

if __name__ == '__main__':

    imagefile = mpimg.imread(
        'C:\\Users\\alyona\\Documents\\cloud\\cloud_app\\FV000046.jpg')

    gray = cv.cvtColor(imagefile, cv.COLOR_BGR2GRAY)

    cascade_path = r'' + cv.data.haarcascades + "haarcascade_frontalface_default.xml"

    face_cascade = cv.CascadeClassifier(cascade_path)
    faces = face_cascade.detectMultiScale(
        gray,
        scaleFactor=1.3,
        minNeighbors=3,
        minSize=(30, 30)
    )

    print("[INFO] Found {0} Faces.".format(len(faces)))

    for (x, y, w, h) in faces:
        cv.rectangle(imagefile, (x, y), (x + w, y + h), (0, 255, 0), 2)
        roi_color = imagefile[y:y + h, x:x + w]
        print("[INFO] Object found. Saving locally.")
        roi_to_save = cv.cvtColor(roi_color, cv.COLOR_BGR2RGB)
        cv.imwrite(str(w) + '_' + str(h) + '_faces.jpg', roi_to_save)

        image = cv.cvtColor(roi_color, cv.COLOR_BGR2RGB)
        img_crop_pil = Img.fromarray(image)
        byte_io = BytesIO()
        img_crop_pil.save(byte_io, 'jpeg')
        jpg_buffer = byte_io.getvalue()
        byte_io.seek(0)

    imagefile_to_save = cv.cvtColor(imagefile, cv.COLOR_BGR2RGB)
    status = cv.imwrite('faces_detected.jpg', imagefile_to_save)
    print("[INFO] Image faces_detected.jpg written to filesystem: ", status)


#!/usr/bin/env python3

import dbxref.resolver
import requests
import logging
import json
import argparse

logger = logging.getLogger(__name__)

def main():
    """main()method for script usage"""
    # AVAILABLE for implementation:
    # 'go_terms', 'member_databases', 'integrated', 'entry_annotations', ''
    #
    # USED:
    # basics: 'accession', 'type', 'description', 'counters', 'entry_id', 'source_database', 'name'
    # hierarchy
    # wikipedia
    # literature
    # cross_references
    # overlaps_with

    parser = argparse.ArgumentParser(description="Retrieve InterPro documents and convert them into json")
    parser.add_argument("--basics", "-b", action="store_true", help="Include basic information such as accession, "
                                                                    "type, name, description, counters, entry_id and "
                                                                    "source_database")
    parser.add_argument("--hierarchy", "-hi", action="store_true", help="")
    parser.add_argument("--wikipedia", "-w", action="store_true", help="")
    parser.add_argument("--literature", "-l", action="store_true", help="")
    parser.add_argument("--cross_references", "-cr", action="store_true", help="")
    parser.add_argument("--overlaps", "-o", action="store_true", help="")
    parser.add_argument("dbxrefs", nargs=argparse.REMAINDER)
    args = parser.parse_args()

    # if nothing specified, output all available information for the entry
    if None not in (args.basics, args.hierarchy, args.wikipedia, args.literature, args.cross_references, args.overlaps):
        args.basics = True
        args.hierarchy = True
        args.wikipedia = True
        args.literature = True
        args.cross_references = True
        args.overlaps = True

    dbxrefs = dbxref.resolver.convert_to_dbxrefs(args.dbxrefs)

    documents = retrieve(dbxrefs, basics=args.basics, hierarchy=args.hierarchy, wikipedia=args.wikipedia,
                         literature=args.literature, cross_references=args.cross_references, overlaps=args.overlaps)
    print(json.dumps(documents, sort_keys=True, indent=4))

def retrieve(dbxrefs, basics=True, hierarchy=True, wikipedia=True, literature=True, cross_references=True, overlaps=True):
    """Retrieve json document from InterPro REST api, filter information by selected Options and parse into new json"""
    resolved = dbxref.resolver.resolve(dbxrefs, check_existence=False)
    documents = []
    for entry in resolved:
        # Construct URL for retrieve
        json_url = entry['locations']['json'][0]
        logger.debug('URL: %s', json_url)
        r = requests.get(json_url)
        logger.debug('Content: %s', r.text)
        ipro = json.loads(r.text)

        # Parse retrieved json file by selected Options
        output = {"id": entry["dbxref"]}
        if basics:
            try:
                output.update(accession=ipro["metadata"]["accession"], entry_type=ipro["metadata"]["type"],
                              description=ipro["metadata"]["description"], counters=ipro["metadata"]["counters"],
                              entry_id=ipro["metadata"]["entry_id"], name=ipro["metadata"]["name"],
                              source_database=ipro["metadata"]["source_database"])
            except KeyError:
                logger.warning("One or more basic information were not available for the given entry. Please check your output.")
        if hierarchy:
            try:
                output.update(hierarchy=ipro["metadata"]["hierarchy"])
            except KeyError:
                logger.warning("Hierarchy information was not available for the given entry.")
        if wikipedia:
            try:
                output.update(wikipedia=ipro["metadata"]["wikipedia"])
            except KeyError:
                logger.warning("Wikipedia articles were not available for the given entry.")
        if literature:
            try:
                output.update(literature=ipro["metadata"]["literature"])
            except KeyError:
                logger.warning("Literature was not available for the given entry.")
        if cross_references:
            try:
                output.update(cross_references=ipro["metadata"]["cross_references"])
            except KeyError:
                logger.warning("Cross_references were not available for the given entry.")
        if overlaps:
            try:
                output.update(overlaps=ipro["metadata"]["overlaps_with"])
            except KeyError:
                logger.warning("Overlap information was not available for the given entry.")
        documents.append(output)
    return documents

if __name__ == "__main__":
    main()


from collections import OrderedDict
ordered_items = OrderedDict()
n = int(input())
name_price = []
for _ in range(n):
    name_price.append(input())

for item in name_price:
    items = item.split()
    item_name = ' '.join(items[:-1])
    item_price = int(items[-1])
    if item_name in ordered_items:
        ordered_items[item_name] += item_price
    else:
        ordered_items[item_name] = item_price

for name,price in ordered_items.items():
    print(name, price)


from decimal import Decimal
from typing import Any, Dict

from pydantic import Field, SecretStr

from hummingbot.client.config.config_data_types import BaseConnectorConfigMap, ClientFieldData
from hummingbot.core.data_type.trade_fee import TradeFeeSchema

DEFAULT_FEES = TradeFeeSchema(
    maker_percent_fee_decimal=Decimal("0.0008"),
    taker_percent_fee_decimal=Decimal("0.0001"),
)

CENTRALIZED = True

EXAMPLE_PAIR = "BTC-USDT"

class OKXConfigMap(BaseConnectorConfigMap):
    connector: str = Field(default="okx", const=True, client_data=None)
    okx_api_key: SecretStr = Field(
        default=...,
        client_data=ClientFieldData(
            prompt=lambda cm: "Enter your OKX API key",
            is_secure=True,
            is_connect_key=True,
            prompt_on_new=True,
        ),
    )
    okx_secret_key: SecretStr = Field(
        default=...,
        client_data=ClientFieldData(
            prompt=lambda cm: "Enter your OKX secret key",
            is_secure=True,
            is_connect_key=True,
            prompt_on_new=True,
        ),
    )
    okx_passphrase: SecretStr = Field(
        default=...,
        client_data=ClientFieldData(
            prompt=lambda cm: "Enter your OKX passphrase key",
            is_secure=True,
            is_connect_key=True,
            prompt_on_new=True,
        ),
    )

KEYS = OKXConfigMap.construct()

def is_exchange_information_valid(exchange_info: Dict[str, Any]) -> bool:
    """
    Verifies if a trading pair is enabled to operate with based on its exchange information

    :param exchange_info: the exchange information for a trading pair

    :return: True if the trading pair is enabled, False otherwise
    """
    return exchange_info.get("instType", None) == "SPOT"


"""
Given a string s, find the length of the longest substring without repeating characters.
"""
from collections import defaultdict

class Solution:
    # O(n^2)
    def lengthOfLongestSubstring(self, s: str) -> int:
        maximum = 0
        for i in range(0, len(s)):
            chars = set()
            for j in range(i, len(s)):
                if s[j] in chars:
                    break
                else:
                    maximum = max(maximum, j-i+1)
                    chars.add(s[j])
        return maximum

    # O(n)
    def lengthOfLongestSubstring(self, s: str) -> int:
        d  = defaultdict(lambda:-1)
        i = 0 # Index of last char that is repeated now
        result = 0 # Max Length os Substring
        for index in range(len(s)):
            i = max(i, d[s[index]]+1)
            result = max(result, index-i+1)
            d[s[index]] = index
        return result


from rest_framework.decorators import api_view
from rest_framework.response import Response
from rest_framework import status

from rest_framework.authtoken.models import Token

from .serializers import RegestrationSerializer
from user_app import models

@api_view(['POST', ])
def logout_view(request):

    if request.method == 'POST':
        request.user.auth_token.delete()
        return Response(status=status.HTTP_200_OK)

@api_view(['POST', ])
def registration_view(request):

    if request.method == 'POST':
        data = request.data
        serializer = RegestrationSerializer(data=data)

        data = {}

        if serializer.is_valid():
            account = serializer.save()       # over-riding save method to add extra field "PASSWORD2"

            # we need to over-ride save method, all the current functionality will be there and we can add some more according to out requirement.

            data['response'] = "Registration Sucessfull."
            data['username'] = account.username
            data['email'] = account.email               # store all this data into data{}

            token = Token.objects.get(user=account).key     # getting token.
            data['token'] = token

        else:
            data = serializer.errors

        return Response(data)


import re
from os import PathLike
from typing import Union, Optional

import aiosqlite

from .. import Notebook
from ..NotebookCell import NotebookCell

class SQLiteNotebook(Notebook):
    def __init__(self, notebook: Union[str, PathLike], execute: bool = False):
        super().__init__(notebook, self.__execute_cell, execute)
        self.db: Optional

    async def __aenter__(self) -> "Notebook":
        self.db = aiosqlite.connect(':memory:')

        await self.db.__aenter__()
        await super().__aenter__()

        return self

    async def __aexit__(self, exc_type, exc_val, exc_tb):
        await super().__aexit__(exc_type, exc_val, exc_tb)
        await self.db.__aexit__(exc_type, exc_val, exc_tb)

    async def __execute_cell(self, cell: NotebookCell):
        statements = list(filter(lambda x: x, re.split(r';$', cell.source, flags=re.MULTILINE)))
        last_index = len(statements) - 1

        for i, statement in enumerate(statements):
            async with self.db.execute(statement) as cursor:
                if i == last_index:
                    return await cursor.fetchall()


from bs4 import BeautifulSoup
import requests
import numpy as np
import csv
from datetime import datetime

print("Welcome to the eBay Price Tracker!")
print("Enter an eBay link to find the average price for the product! ")
LINK = input("Input here: ")

def get_prices_by_link(link):
    r = requests.get(link)
    page_parse = BeautifulSoup(r.text, 'html.parser')
    search_results = page_parse.find("ul", {"class": "srp-results"}).find_all("li", {"class": "s-item"})
    item_prices = []

    for result in search_results:
        price_as_text = result.find("span", {"class": "s-item__price"}).text
        if "to" in price_as_text:
            continue
        price = float(price_as_text[1:].replace(",", ""))
        item_prices.append(price)
    return item_prices

def getProductName(link):
    r = requests.get(link)
    page_parse = BeautifulSoup(r.text, 'html.parser')
    input_tag = page_parse.find("input", {"class": "gh-tb ui-autocomplete-input"})
    product_name = input_tag["value"]
    return product_name

def remove_outliers(prices, m=2):
    data = np.array(prices)
    return data[abs(data - np.mean(data)) < m * np.std(data)]

def get_avg(prices):
    return np.mean(prices)

def save_to_file(prices):
    fields = ["Date: "+datetime.today().strftime("%d-%m-%y"),
              "Product name: "+getProductName(LINK),
              "Average price: "+str(np.around(get_avg(prices), 2))]
    with open('prices.csv', 'a', newline='') as f:
        writer = csv.writer(f)
        writer.writerow(fields)

if __name__ == "__main__":
    prices = get_prices_by_link(LINK)
    without_outliers = remove_outliers(prices)
    print("Product name: "+getProductName(LINK))
    print("Average Price: "+str(np.around(get_avg(without_outliers), 2)))
    save_to_file(without_outliers)


# -*- coding: UTF-8 -*-

import pygame
import glob
import datetime
import os

class animatedPng(object):
    def __init__(self, renderingSurface, folderpath, pause, resize = None, ratio = None, colorkey = (0,0,0)):
        #a class to animate sets of png's
        self.images = []

        savedPath = os.getcwd()
        os.chdir(folderpath)
        #gets all pngs's in the given path, converts, resizes and saves them in self.images
        for filename in glob.glob('*.png'):
            buffer = pygame.image.load(filename).convert()

            if resize != None:
                buffer = pygame.transform.scale(buffer, resize)
                rect = pygame.Rect((0,0), resize)
            elif ratio != None:
                rect = buffer.get_rect()
                rect.width *= ratio[1]
                rect.height *= ratio[1]

                rect.height = int(rect.height)
                rect.width = int(rect.width)

                buffer = pygame.transform.scale(buffer, (rect.width, rect.height))

            buffer.set_colorkey(colorkey)
            self.images.append(buffer)
        os.chdir(savedPath)

        self.renderingSurface = renderingSurface

        self.pause = datetime.timedelta(milliseconds = pause)
        self.lastshown = datetime.datetime.now() #time of last picture change

        self.index = 0
        self.count = len(self.images)

        self.size = rect.width, rect.height

    def render(self, x, y):
        #if the last picture in list is reaged then start over
        if self.index == self.count:
            self.index = 0
        #blits the image on the renderingsurface at the given coordinates
        self.renderingSurface.blit(self.images[self.index], (x, y))

        #jump to the next image in list, when the pause is equal or higher the given one between to images
        now = datetime.datetime.now()
        if now - self.lastshown >= self.pause:
            self.index += 1
            self.lastshown = now


# bot-term.py
# A simple interactive terminal to test the model
# Usage: python3 bot-term.py
# ITL 2020-11-15

import requests

# api_url = "http://3.139.62.97:4000/predict"
api_url = "http://ec2-3-139-62-97.us-east-2.compute.amazonaws.com:4000/predict"

print("\nWelcome to the Accurolab Covid-19 test bot.")
print("Please ask a question, or hit return to exit.\n")

continue_loop = True
while continue_loop:
  q = input("> ")
  if q == "":
    continue_loop = False
  else:
    payload = {"question" : q }
    r = requests.put(api_url, json=payload).json()
    for x in r:
        if x != 'question':
            print(x,": ",r[x])
    print("\n")


import sys
from collections import deque
import copy
input = sys.stdin.readline
inf = sys.maxsize

n, m = map(int, input().split())
board = [list(map(int, input().split())) for _ in range(n)]

dr = [1, -1, 0, 0]
dc = [0, 0, -1, 1]

virus_cnt = 0   # 처음 바이러스의 개수
virus_loc = []  # 바이러스의 위치
empty_cnt = 0   # 빈칸의 개수
for i in range(n):
    for j in range(n):
        if board[i][j] == 2:
            virus_cnt += 1
            virus_loc.append([i, j, 0])
        elif board[i][j] == 0:
            empty_cnt += 1

pick = deque()
ccc= 0
ans = inf
def bfs(q, visited):
    global ans
    # 비어있는 칸에 몇초 걸려서 감염시키는지
    while q:
        r, c, time = q.popleft()
        # if time == ans:
        #     for i in visited:
        #         print(i)
        #     return
        for i in range(4):
            nr = r + dr[i]
            nc = c + dc[i]
            if 0 <= nr < n and 0 <= nc < n and board[nr][nc] != 1:
                if visited[nr][nc] > time + 1:
                    visited[nr][nc] = time + 1
                    q.append([nr, nc, time + 1])

    spread_cnt = 0 # 몇칸 감염시켰는지
    max_time = 0 # 가장 오래걸린 시간
    # for i in visited:
    #     print(i)
    for i in range(n):
        for j in range(n):
            if visited[i][j] != inf and board[i][j] == 0:
                if visited[i][j] > max_time:
                    max_time = visited[i][j]
                spread_cnt += 1
    # print(max_time)
    # 만약 감염시킨 칸의 개수가 처음 빈칸의 개수와 같을때
    if spread_cnt == empty_cnt:
        if max_time < ans:
            ans = max_time
        return

def combi(cnt, start):
    global virus_cnt, m, ccc
    # 바이러스를 m개 뽑는 조합
    if cnt == m:
        ccc += 1
        visited = [[inf]*n for _ in range(n)]
        for loc in pick:
            visited[loc[0]][loc[1]] = 0
        # print(pick)
        q = copy.deepcopy(pick)
        bfs(q, visited)
        return
    for i in range(start, virus_cnt):
        pick.append(virus_loc[i])
        combi(cnt + 1, i + 1)
        pick.pop()

combi(0, 0)
if ans == inf:
    print(-1)
else:
    print(ans)


import os
import re
import json
import requests
from file import write_text, create_dir
import argparse

def download(url):
    return requests.get(url).text

class ReactComponentNotFound(RuntimeError):
    pass

class ReactSpider(object):
    @staticmethod
    def extract_react_component(page_source):
        """
        Extract data used to initialize `ReactComponent` from the given page source
        This will only work if the `ReactComponent` is constructed in one line
        """
        regex = re.search('ReactComponent\((.+,\s*"loggedIn".+?})', page_source)
        if not regex:
            raise ReactComponentNotFound(page_source)
        return json.loads(regex.group(1))

    @classmethod
    def crawl(cls, url):
        page_source = download(url)
        react_component = cls.extract_react_component(page_source)
        return cls.filter(react_component)

    @classmethod
    def filter(cls, component):
        """
        filter and return necessary data from react component
        :param component: unfiltered data used to initialize `ReactComponent`
        :return: filtered data
        """
        raise NotImplemented()

class TutorialSpider(ReactSpider):
    @classmethod
    def filter_content_items(cls, content_items):
        result = []
        for ci in content_items:
            # practice nodes do not have videos
            if 'youtubeId' not in ci:
                continue
            result.append({'title': ci['title'], 'video_id': ci['youtubeId']})
        return result

    @classmethod
    def filter(cls, component):
        tutorials = component['componentProps']['curation']['tabs'][0]['modules'][0]['tutorials']
        nodes = []
        for t in tutorials:
            nodes.append({
                'title': t['title'],
                'children': cls.filter_content_items(t['contentItems'])})
        return nodes

class ModuleSpider(ReactSpider):
    @classmethod
    def filter(cls, component):
        modules = component['componentProps']['curation']['tabs'][0]['modules']
        modules = [m for m in modules if m['kind'] == 'TableOfContentsRow']
        return [{'title': m['title'], 'url': m['url']} for m in modules]

def get_url_base(url):
    regex = re.search('(.+://.+?)/.+', url)
    return regex.group(1)

def adjust_module_urls(modules, url):
    """Convert relative paths to full urls"""
    url_base = get_url_base(url)
    for m in modules:
        m['url'] = url_base + m['url']
    return modules

def get_modules(url):
    modules = ModuleSpider.crawl(url)
    return adjust_module_urls(modules, url)

def save(title, modules):
    out_dir = os.path.join('transcripts', 'tutorials')
    create_dir(out_dir)
    fname = os.path.join(out_dir, title + '.json')
    write_text(fname, json.dumps(modules))

def convert_hierarchy(hierarchy):
    """Convert dict hierarchy to directory hierarchy"""
    for c in hierarchy:
        for gc in c['children']:
            yield {'path': os.path.join(c['title'], gc['title']), 'video_id': gc['video_id']}

def commit_hierarchy(hierarchy):
    """
    Commit hierarchy to file system so that they can be cached
    :param hierarchy: hierarchy to commit
    :param dest: directory in file system to commit to
    """
    for node in hierarchy:
        create_dir(os.path.dirname(node['path']))
        write_text(node['path'], node['video_id'])

def resolve_paths(hierarchy, dest):
    """
    Resolve relative paths in hierarchy
    :param hierarchy: hierarchy containing video ids; mutated
    :param dest: Destination in file system to save hierarchy
    :return: hierarchy with resolved paths
    """
    for node in hierarchy:
        node['path'] = os.path.join(dest, node['path'])
    return hierarchy

def resolve_indices(hierarchy):
    """
    Prepend index of array element to title so that the order is
    preserved after saving to the file system
    :param hierarchy: hierarchy containing video ids; mutated
    :return: hierarchy with resolved indices
    """
    for i, h in enumerate(hierarchy):
        h['title'] = '%d %s' % (i, h['title'])
        if 'children' in h:
            resolve_indices(h['children'])
    return hierarchy

def fix_illegal_names(hierarchy):
    """
    Fix illegal names
    :param hierarchy: hierarchy containing video ids; mutated
    :return: hierarchy with resolved indices
    """
    for i, h in enumerate(hierarchy):
        h['title'] = h['title'].replace('/', '_').replace(':', '_')
        if 'children' in h:
            fix_illegal_names(h['children'])
    return hierarchy

def process_modules(url, dest_dir):
    for m_i, m in enumerate(get_modules(url)):
        module_title = '%d %s' % (m_i, m['title'])
        print('\tProcessing ' + module_title)

        module_dir = os.path.join(dest_dir, module_title)

        if os.path.exists(module_dir):
            print('\t\tSkipping ' + module_dir)
            continue

        hierarchy = TutorialSpider.crawl(m['url'])
        hierarchy = resolve_indices(hierarchy)
        hierarchy = fix_illegal_names(hierarchy)
        hierarchy = list(convert_hierarchy(hierarchy))
        hierarchy = resolve_paths(hierarchy, module_dir)
        commit_hierarchy(hierarchy)

        print('\tSaved ' + module_title)

def get_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('module')
    parser.add_argument('url')
    return parser.parse_args()

def main():
    """
    Download tutorials of every content in every module
    eg) Vectors and spaces(module) => Vectors(content) => Vector intro for linear algebra(tutorial)
    """
    args = get_args()

    module_title = args.module

    print('Processing ' + module_title)
    subject_dir = os.path.join('video_ids', module_title)
    process_modules(args.url, subject_dir)

if __name__ == '__main__':
    main()


from abc import ABCMeta, abstractmethod
import numbers
import warnings

import torch

from ignite.engine import State, Engine
from ignite._six import with_metaclass

class BaseLogger(object):
    """
    Base logger handler. See implementations: TensorboardLogger, VisdomLogger, PolyaxonLogger

    """
    def attach(self, engine, log_handler, event_name):
        """Attach the logger to the engine and execute `log_handler` function at `event_name` events.

        Args:
            engine (Engine): engine object.
            log_handler (callable): a logging handler to execute
            event_name: event to attach the logging handler to. Valid events are from :class:`~ignite.engine.Events`
                or any `event_name` added by :meth:`~ignite.engine.Engine.register_events`.

        """
        if event_name not in State.event_to_attr:
            raise RuntimeError("Unknown event name '{}'".format(event_name))

        engine.add_event_handler(event_name, log_handler, self, event_name)

    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        self.close()

    def close(self):
        pass

class BaseHandler(with_metaclass(ABCMeta, object)):

    @abstractmethod
    def __call__(self, *args, **kwargs):
        pass

class BaseOptimizerParamsHandler(BaseHandler):
    """
    Base handler for logging optimizer parameters
    """

    def __init__(self, optimizer, param_name="lr", tag=None):
        if not isinstance(optimizer, torch.optim.Optimizer):
            raise TypeError("Argument optimizer should be of type torch.optim.Optimizer, "
                            "but given {}".format(type(optimizer)))

        self.optimizer = optimizer
        self.param_name = param_name
        self.tag = tag

class BaseOutputHandler(BaseHandler):
    """
    Helper handler to log engine's output and/or metrics
    """

    def __init__(self, tag, metric_names=None, output_transform=None, another_engine=None, global_step_transform=None):

        if metric_names is not None and not isinstance(metric_names, list):
            raise TypeError("metric_names should be a list, got {} instead.".format(type(metric_names)))

        if output_transform is not None and not callable(output_transform):
            raise TypeError("output_transform should be a function, got {} instead."
                            .format(type(output_transform)))

        if output_transform is None and metric_names is None:
            raise ValueError("Either metric_names or output_transform should be defined")

        if another_engine is not None:
            if not isinstance(another_engine, Engine):
                raise TypeError("Argument another_engine should be of type Engine, "
                                "but given {}".format(type(another_engine)))
            warnings.warn("Use of another_engine is deprecated and will be removed in 0.2.1. "
                          "Please use global_step_transform instead.", DeprecationWarning)

        if global_step_transform is not None and not callable(global_step_transform):
            raise TypeError("global_step_transform should be a function, got {} instead."
                            .format(type(global_step_transform)))

        if global_step_transform is None:
            def global_step_transform(engine, event_name):
                return engine.state.get_event_attrib_value(event_name)

        self.tag = tag
        self.metric_names = metric_names
        self.output_transform = output_transform
        self.another_engine = another_engine
        self.global_step_transform = global_step_transform

    def _setup_output_metrics(self, engine):
        """Helper method to setup metrics to log
        """
        metrics = {}
        if self.metric_names is not None:
            for name in self.metric_names:
                if name not in engine.state.metrics:
                    warnings.warn("Provided metric name '{}' is missing "
                                  "in engine's state metrics: {}".format(name, list(engine.state.metrics.keys())))
                    continue
                metrics[name] = engine.state.metrics[name]

        if self.output_transform is not None:
            output_dict = self.output_transform(engine.state.output)

            if not isinstance(output_dict, dict):
                output_dict = {"output": output_dict}

            metrics.update({name: value for name, value in output_dict.items()})
        return metrics

class BaseWeightsScalarHandler(BaseHandler):
    """
    Helper handler to log model's weights as scalars.
    """

    def __init__(self, model, reduction=torch.norm):
        if not isinstance(model, torch.nn.Module):
            raise TypeError("Argument model should be of type torch.nn.Module, "
                            "but given {}".format(type(model)))

        if not callable(reduction):
            raise TypeError("Argument reduction should be callable, "
                            "but given {}".format(type(reduction)))

        def _is_0D_tensor(t):
            return isinstance(t, torch.Tensor) and t.ndimension() == 0

        # Test reduction function on a random tensor
        o = reduction(torch.rand(4, 2))
        if not (isinstance(o, numbers.Number) or _is_0D_tensor(o)):
            raise ValueError("Output of the reduction function should be a scalar, but got {}".format(type(o)))

        self.model = model
        self.reduction = reduction

class BaseWeightsHistHandler(BaseHandler):
    """
    Helper handler to log model's weights as histograms.
    """

    def __init__(self, model):
        if not isinstance(model, torch.nn.Module):
            raise TypeError("Argument model should be of type torch.nn.Module, "
                            "but given {}".format(type(model)))

        self.model = model


#!/usr/bin/env python

"""
Filename:  transformer_classifier.py
Description:
    This system fine-tunes an Italian BERT model to predict who (out of three subjects) is the
    post-editor of an automatically translated sentence. Predictions are cross-validated on the training set.
"""

import argparse
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split, StratifiedKFold
from simpletransformers.classification import ClassificationArgs, ClassificationModel

def create_arg_parser():
    parser = argparse.ArgumentParser()
    parser.add_argument("-tr", "--train_file", default='./data/IK_NLP_22_PESTYLE/train.tsv',
                        type=str, help="Location of training file.")
    parser.add_argument("-te", "--test_file", default='./data/IK_NLP_22_PESTYLE/test.tsv',
                        type=str, help="Location of test file.")
    parser.add_argument("-o", "--outfile", default='./transformer_predictions.txt',
                        type=str, help="Path to write output to.")
    parser.add_argument("-m", "--model",
                        type=str, help="Specify an already fine-tuned model")
    return parser.parse_args()

def prepare_train(train_file, include_from_scratch=False):
    """
    Extract the relevant features from the data set: MT, PE and subject.
    If include_from_scratch is False, we do not take into account the HT modality.
    """
    pairs = []
    labels = []
    with open(train_file) as train:
        train.readline()  # Skip first line: this is the header
        for line in train:
            if line.split("\t")[1][0] == "t":
                # Get a list of: [MT output, post-edited sentence, subject id]
                if include_from_scratch:
                    pairs.append([line.split("\t")[4], line.split("\t")[5]])
                    labels.append(line.split("\t")[1])
                else:
                    if line.split("\t")[4]:
                        pairs.append([line.split("\t")[4], line.split("\t")[5]])
                        labels.append(line.split("\t")[1])

    # We use only 90% of the data, in order to allow comparison with the other models
    X_train, _, y_train, _ = train_test_split(pairs, labels, test_size=0.1, random_state=42)

    return X_train, y_train

def predict_test(test_data, model, include_from_scratch=False, final=False):
    """
    Given a fine-tuned model, predict the labels (translators) from MT-PE sentence pairs.
    If include_from_scratch is False, we do not take into account the HT modality.
    """
    pairs = []
    predictions = []

    if final:  # Use the 'final' (official) test set
        with open(test_data) as test:
            test.readline()  # Skip first line: this is the header
            for line in test:
                if line.split("\t")[0][0] in "0123456789":
                    if include_from_scratch:
                        pairs.append([line.split("\t")[4], line.split("\t")[5]])
                    else:
                        if line.split("\t")[4]:
                            pairs.append([line.split("\t")[4], line.split("\t")[5]])
    else:  # Use a subset of the data for cross-validation
        pairs = test_data

    for pair in pairs:
        prediction, _ = model.predict(pair)
        predictions.append(prediction[0])

    return predictions

def get_accuracy(gold, pred):
    """
    Retrieve the number of correctly predicted post-editors
    """
    # Map labels to ints, to streamline with model predictions
    gold_ints = []
    for g in gold:
        if g == "t1":
            gold_ints.append(0)
        elif g == "t2":
            gold_ints.append(1)
        elif g == "t3":
            gold_ints.append(2)
        else:
            pass

    # Get number of correct predictions
    correct = 0
    if len(gold_ints) == len(pred):
        for g, p in zip(gold_ints, pred):
            if g == p:
                correct += 1
    else:
        print("ERROR: Gold standard and predictions are not of equal length!")

    return correct / len(gold)

def reformat_test(pairs):
    """
    Retrieve a regular list from the numpy array
    The output is a list that contains lists ([MT,PE]) with each sentence pair.
    """
    data = []
    mt = [t[0] for t in pairs]
    pe = [t[1] for t in pairs]

    for m, p in zip(mt, pe):
        data.append([m, p])

    return data

def reformat_train(pairs, labels):
    """
    Retrieve a regular list from the numpy array
    The output is a list that contains lists ([MT,PE,label]) from each sentence pair.
    """
    data = []
    mt = [t[0] for t in pairs]
    pe = [t[1] for t in pairs]
    labs = np.ndarray.tolist(labels)

    for m, p, l in zip(mt, pe, labs):
        data.append([m, p, l])

    return data

def main():

    args = create_arg_parser()
    all_predictions = []

    if args.model:  # Use an already fine-tuned model, test on the final test set
        model = ClassificationModel("bert", args.model, use_cuda=False)
        predictions = predict_test(args.test_file, model, final=True)
        all_predictions.append(predictions)
        all_predictions.append("\n")

    else:  # Fine-tune a new model, use cross-validation

        # Define model
        model_args = ClassificationArgs()
        model_args.num_train_epochs = 3
        model_args.overwrite_output_dir = True
        model_args.learning_rate = 0.000001
        model_args.do_lower_case = True
        model_args.silence = True
        model = ClassificationModel("bert", "dbmdz/bert-base-italian-uncased", args=model_args, use_cuda=False, num_labels=3)

        # Create the correct splits for cross-validation
        skf = StratifiedKFold(n_splits=10, shuffle=True, random_state=42)
        X, y = prepare_train(args.train_file)
        X = np.array(X)
        y = np.array(y)
        for train_index, test_index in skf.split(X, y):
            X_train, X_test = X[train_index], X[test_index]
            y_train, y_test = y[train_index], y[test_index]

            # Prepare train set
            train_data = reformat_train(X_train, y_train)
            train_df = pd.DataFrame(train_data)
            train_df.columns = ["text_a", "text_b", "labels"]
            train_df["labels"] = train_df["labels"].replace(["t1", "t2", 't3'], [0, 1, 2])
            print(train_df)

            # Train model
            model.train_model(train_df)

            # Predict synsets
            predictions = predict_test(reformat_test(X_test), model)
            all_predictions.append(predictions)
            all_predictions.append("\n")
            print("ACCURACY: ", get_accuracy(y_test, predictions))

    # Write results to pickle file
    with open(args.outfile, 'w') as pred_file:
        for pred_list in all_predictions:
            for pred in pred_list:
                pred_file.write(str(pred))
    print("Predictions have been written to file: " + args.outfile)

if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-
"""
Created on Thu Oct 22 11:29:22 2020

@author: Family
"""

import cv2
import numpy as np
from scipy import ndimage

img1 = cv2.imread('images/bananos.jpg') #Ori image
#img1 = cv2.imread('images/A.jpg') #Ori image
#img1 = cv2.imread('images/B.jpg') #Ori image
#img1 = cv2.imread('images/car1.jpg') #Ori image
#img1 = cv2.imread('images/car3.jpg') #Ori image
#img1 = cv2.imread('images/lunar_cancer.jpg') #Ori image
I = cv2.cvtColor(img1,cv2.COLOR_BGR2GRAY) #Gray image
umbral,_ = cv2.threshold(I,0,255,cv2.THRESH_OTSU) #Get umbral
mascara = np.uint8((I<umbral)*255) #Binary image

#cv2.imshow("Img-COLOR-ori",img1)
#cv2.imshow("Img-GRAY",I)
#cv2.imshow("Img-BINARY",mascara)

#Get labels
output = cv2.connectedComponentsWithStats(mascara,4,cv2.CV_32S)
cantObj = output[0] # Objects quantity
labels = output[1] # Labels
stats = output[2]

#Get ArgMax
mascara_limpia = (np.argmax(stats[:,4][1:])+1==labels)

#Rellenar huecos de la nueva máscara
#Esto nos permite a futuro obtener valores como el área, perímetro, etc.
new_mascara = ndimage.binary_fill_holes(mascara_limpia).astype(int)

#Medición del área y perímetro en la imágen.
last_mascara = np.uint8(new_mascara*255)
contours,_ = cv2.findContours(last_mascara, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) #Sacar los contornos
cnt = contours[0] # Puntos específicos
Perimetro = cv2.arcLength(cnt, False)

#Area_man = np.sum(last_mascara/255)
Area = cv2.contourArea(cnt)

#ConvexHull => Polígono convexo
hull = cv2.convexHull(cnt) # Obtener puntos del polígono que se está creando
puntoConvex = hull[:,0,:] # Generar el punto convexo
m,n = last_mascara.shape # Genero Dimensiones de la imagen
ar = np.zeros((m,n))
mascara_convex = np.uint8(cv2.fillConvexPoly(ar,puntoConvex,1)) # 1: Grosor de línea

#BoundingBox
#=>BoundingBox => Rotado (ángulos)
rect = cv2.minAreaRect(cnt)
box = np.int0(cv2.boxPoints(rect))
m,n = mascara_convex.shape
ar = np.zeros((m,n))
mascara_rectang = np.uint8(cv2.fillConvexPoly(ar,box,1))

#=>BoundingBox Recto =>
x, y, w, h = cv2.boundingRect(cnt)
#ractangle(img_ori, vertices, ancho y alto, color de línea, grosor de línea)
cv2.rectangle(img1, (x,y), (x+w, y+h), (255,0,0) , 1)

contours,_ = cv2.findContours(mascara_convex, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) #Sacar los contornos
cv2.drawContours(img1, contours, -1, (0,255,0), 1)

contours,_ = cv2.findContours(mascara_rectang, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) #Sacar los contornos
cv2.drawContours(img1, contours, -1, (0,0,255), 1)

cv2.imshow("Img-COLOR-Bounding",img1)

#Segmentar el objeto de interés => Aislar el fondo del objeto a procesar.

segmentoColor = np.zeros((m,n,3)).astype(np.uint8)

#Color
segmentoColor[:,:,0] = np.uint8(img1[:,:,0]*new_mascara)
segmentoColor[:,:,1] = np.uint8(img1[:,:,1]*new_mascara)
segmentoColor[:,:,2] = np.uint8(img1[:,:,2]*new_mascara)
#gray
segmentoGray = np.zeros((m,n))
segmentoGray[:,:] = np.uint8(I*new_mascara)

cv2.imshow('Img-segmento-color', segmentoColor)
cv2.imshow('Img-segmento-gray', segmentoGray)

cv2.waitKey(0)
cv2.destroyAllWindows()


from PyQt5.QtWidgets import QApplication, QWidget
from PyQt5.QtWidgets import QPushButton, QLabel, QComboBox, QCheckBox, QTableWidget, QTableWidgetItem
from PyQt5.QtWidgets import QLineEdit, QDateEdit, QTimeEdit
from PyQt5.QtCore import QDate, QTime, QSize
from PyQt5.QtGui import QIcon, QFont
from gui_atividades import *
import sys

class WindowAudiencias(QWidget):

    def __init__(self, db, parent=None):
        super(WindowAudiencias, self).__init__(parent)
        self.db = db
        self.setWindowIcon(QIcon("static/logo.ico"))
        self.setWindowTitle("Audiências do Tribunal de Justiça")
        self.resize(800, 600)
        self.filtered_date = False
        self.filtered_vara = False

        # buttons
        x_btn = 600
        y_btn_gap = 20
        y_btn_step = 50

        self.btn_font = QFont()
        self.btn_font.setBold(True)
        self.btn_font.setPixelSize(12)

        def create_button(window, on_click, text, tip, pos):
            button = QPushButton(text, window)
            button.setToolTip(tip)
            button.move(x_btn, y_btn_gap + y_btn_step * pos)
            button.clicked.connect(on_click)
            button.resize(150, 40)
            button.setFont(self.btn_font)
            return button

        self.btn_insert_audiencia = create_button(self, self.on_click_insert_audiencia,
            'Inserir Audiência', 'Criar uma nova Audiência no banco de dados.', 0)
        self.btn_update_audiencia = create_button(self, self.on_click_update_audiencia,
            'Atualizar Audiência', 'Atualizar uma Audiência do banco de dados.', 1)
        self.btn_delete_audiencia = create_button(self, self.on_click_delete_audiencia,
            'Remover Audiência', 'Remover uma Audiência do banco de dados.', 2)
        self.btn_atividades = create_button(self, self.on_click_open_atividades,
            'Atividades Audiência', 'Abrir a janela de atividades da Audiência.', 3)

        # filter buttons
        def create_filter_button(window, on_click, icon, tip, x):
            filter_button = create_button(window, on_click, '', tip, 3.5)
            filter_button.setIcon(QIcon('./static/{}'.format(icon)))
            filter_button.resize(40, 40)
            filter_button.move(x, y_btn_gap + y_btn_step * 3.5)
            filter_button.setIconSize(QSize(30, 30) )
            return filter_button

        self.btn_filter_vara = create_filter_button(self, self.on_click_filter_vara,
            'filter_vara.png', 'Filtrar Audiências pela Vara.', 300)
        self.btn_filter_date = create_filter_button(self, self.on_click_filter_date,
            'filter_data.png', 'Filtrar Audiências por data da Audiência.', 350)
        self.btn_remove_filter_date = create_filter_button(self, self.on_click_remove_filters,
            'rm_filter.png', 'Remover filtros das Audiências.', 400)

        # labels
        def create_label(window, text, x, pos):
            label = QLabel(window)
            label.setText(text)
            y_gap = 20
            y_step = 50
            label.move(x, y_gap + pos * y_step)
            return label

        x_in_1 = 50
        x_in_2 = 220

        create_label(self, 'Vara', x_in_1, 0)
        create_label(self, 'Processo', x_in_1, 1)
        create_label(self, 'Tipo', x_in_1, 2)
        create_label(self, 'Audiências', x_in_1, 4)

        create_label(self, 'Horário', x_in_2, 0)
        create_label(self, 'Data Audiência', x_in_2, 1)
        create_label(self, 'Data Despacho', x_in_2, 2)

        # line edit
        y_gap = 37
        y_step = 50

        def create_line_edit(window, x, pos):
            line_edit = QLineEdit(window)
            line_edit.move(x, y_gap + pos * y_step)
            line_edit.resize(150, 24)
            return line_edit

        self.line_processo = create_line_edit(self, x_in_1, 1)

        # date edit
        def create_date_edit(window, x, pos):
            date_edit = QDateEdit(self)
            date_edit.move(x, y_gap + pos * y_step)
            date_edit.resize(150, 24)
            return date_edit

        self.date_data_audiencia = create_date_edit(self, x_in_2, 1)
        self.date_data_despacho = create_date_edit(self, x_in_2, 2)

        # time edit
        self.time_horario = QTimeEdit(self)
        self.time_horario.move(x_in_2, y_gap + 0 * y_step)
        self.time_horario.resize(150, 24)

        # combo-box
        def create_cb(window, list_items, x, pos):
            combo_box = QComboBox(window)
            combo_box.addItems(list_items)
            combo_box.move(x, y_gap + pos * y_step)
            combo_box.resize(150, 24)
            return combo_box

        self.cb_vara = create_cb(self, ["1", "2"], x_in_1, 0)
        self.cb_tipo = create_cb(self, ["Instrução e Julgamento", "Oitiva", "Interrogatório", "Depoimento Especial", "UNA", "Continuação", "Carta Precatória"], x_in_1, 2)

        # check-box
        self.ck_urgente = QCheckBox("Urgente", self)
        self.ck_urgente.move(x_in_1, y_gap-17 + 3 * y_step)

        # table
        self.table_audiencias = QTableWidget(self)
        self.table_audiencias.move(50, 240)
        self.table_audiencias.resize(700, 300)
        self.table_audiencias.itemSelectionChanged.connect(self.table_audiencias_select)
        self.set_table_audiencias_content(self.filtered_date, self.filtered_vara)

    # table functions
    def set_table_audiencias_content(self, filtered_date, filtered_vara):
        audiencias = self.db.get_audiencias()
        self.table_audiencias.clear()

        headers = list(self.db.get_audiencia_columns())
        n_rows = len(audiencias)
        self.table_audiencias.setRowCount(n_rows)
        if (n_rows <= 0):
            return

        self.table_audiencias.setColumnCount(len(audiencias[0]))
        self.table_audiencias.setHorizontalHeaderLabels(headers)

        ignored_rows = []
        for row, record in enumerate(audiencias):
            ignore_row = False
            for column, content in enumerate(record):
                self.table_audiencias.setItem(row, column, QTableWidgetItem(content))

                if (filtered_date):
                    if (headers[column] == 'data_audiencia' and content != self.get_date_data_audiencia()):
                        if (row not in ignored_rows):
                            ignored_rows.append(row)

                if (filtered_vara):
                    if (headers[column] == 'vara' and content != self.get_cb_vara()):
                        if (row not in ignored_rows):
                            ignored_rows.append(row)

        for i, row in enumerate(ignored_rows):
            self.table_audiencias.removeRow(row-i)

    def table_audiencias_select(self):
        currentRow = self.table_audiencias.currentRow()
        self.table_audiencias.selectRow(currentRow)

        vara_index = self.cb_vara.findText(self.table_audiencias.item(currentRow, 0).text())
        self.cb_vara.setCurrentIndex(vara_index)

        tipo_index = self.cb_tipo.findText(self.table_audiencias.item(currentRow, 2).text())
        self.cb_tipo.setCurrentIndex(tipo_index)

        status_urgente = self.table_audiencias.item(currentRow, 3).text()
        status_urgente = 2 if status_urgente == 'Sim' else 0
        self.ck_urgente.setCheckState(status_urgente)

        txt_horario = self.table_audiencias.item(currentRow, 4).text()
        self.time_horario.setTime(self.transform_to_qtime(txt_horario))

        select_processo = self.table_audiencias.item(currentRow, 1)
        select_data_audiencia = self.table_audiencias.item(currentRow, 5)

        if (select_processo and select_data_audiencia):
            select_processo = select_processo.text()
            self.line_processo.setText(select_processo)

            date_to_set = self.transform_to_qdate(select_data_audiencia.text())
            self.date_data_audiencia.setDate(date_to_set)

        select_data_despacho = self.table_audiencias.item(currentRow, 6)
        if (select_data_despacho):
            date_to_set = self.transform_to_qdate(select_data_despacho.text())
            self.date_data_despacho.setDate(date_to_set)

    # get data functions
    def transform_to_qdate(self, date_str):
        date_list = date_str.split('/')
        return QDate(int(date_list[2]), int(date_list[1]), int(date_list[0]))

    def transform_to_qtime(self, time_str):
        time_list = time_str.split(':')
        return QTime(int(time_list[0]), int(time_list[1]))

    def get_ck_urgente(self):
        return self.ck_urgente.checkState()

    def get_cb_vara(self):
        return self.cb_vara.currentText()

    def get_cb_tipo(self):
        return self.cb_tipo.currentText()

    def get_line_processo(self):
        return self.line_processo.text()

    def get_time_horario(self):
        return self.time_horario.text()

    def get_date_data_audiencia(self):
        return self.date_data_audiencia.text()

    def get_date_data_despacho(self):
        return self.date_data_despacho.text()

    # button functions
    def on_click_filter_date(self):
        self.filtered_date = True
        self.set_table_audiencias_content(self.filtered_date, self.filtered_vara)

    def on_click_filter_vara(self):
        self.filtered_vara = True
        self.set_table_audiencias_content(self.filtered_date, self.filtered_vara)

    def on_click_remove_filters(self):
        self.filtered_date = False
        self.filtered_vara = False
        self.set_table_audiencias_content(self.filtered_date, self.filtered_vara)

    def on_click_insert_audiencia(self):
        self.db.insert_audiencia(vara=self.get_cb_vara(),
                                 processo=self.get_line_processo(),
                                 tipo=self.get_cb_tipo(),
                                 urgente=self.get_ck_urgente(),
                                 horario=self.get_time_horario(),
                                 data_audiencia=self.get_date_data_audiencia(),
                                 data_despacho=self.get_date_data_despacho())
        self.set_table_audiencias_content(self.filtered_date, self.filtered_vara)

    def on_click_delete_audiencia(self):
        currentRow = self.table_audiencias.currentRow()
        drop_processo = self.table_audiencias.item(currentRow, 1)
        drop_data_audiencia = self.table_audiencias.item(currentRow, 5)

        if (drop_processo and drop_data_audiencia):
            drop_processo = drop_processo.text()
            drop_data_audiencia = drop_data_audiencia.text()
            self.db.delete_audiencia(drop_processo, drop_data_audiencia)

            print('Audiência removida do banco de dados.')
            self.set_table_audiencias_content(self.filtered_date, self.filtered_vara)

    def on_click_update_audiencia(self):
        currentRow = self.table_audiencias.currentRow()
        update_processo = self.table_audiencias.item(currentRow, 1)
        update_data_audiencia = self.table_audiencias.item(currentRow, 5)

        if (update_processo and update_data_audiencia):
            update_processo = update_processo.text()
            update_data_audiencia = update_data_audiencia.text()
            self.db.update_audiencia(vara=self.get_cb_vara(),
                                     processo=update_processo,
                                     tipo=self.get_cb_tipo(),
                                     urgente=self.get_ck_urgente(),
                                     horario=self.get_time_horario(),
                                     data_audiencia=update_data_audiencia,
                                     data_despacho=self.get_date_data_despacho())
            self.set_table_audiencias_content(self.filtered_date, self.filtered_vara)

    def on_click_open_atividades(self):
        use_processo = self.get_line_processo()
        use_data_audiencia = self.get_date_data_audiencia()

        if (len(use_processo) != 13 or len(use_data_audiencia) != 10):
            print('Audiência selecionada é inválida.')
        else:
            self.windowAtividades = WindowAtividades(self.db, use_processo, use_data_audiencia, self.get_cb_vara())
            self.windowAtividades.show()


from tabulate import tabulate

def imprimeTablaComplejidad(tipo, comparaciones, intercambios, realizadas, tiempo):
    """Crea la tabla de complejidades recibiendo 3 listas con 2 elementos  y un parametro de tiempo."""

    headers = [" ", "COMPARACIONES", tipo]
    data1 = [("Notación O", comparaciones[0], intercambios[0]),
             ("Complejidad", comparaciones[1], intercambios[1]),
             ]

    data2 = [("Realizadas", realizadas[0], realizadas[1])]
    print(tabulate(data1, headers=headers, tablefmt="fancy_grid"))
    print(tabulate(data2, headers=headers, tablefmt="fancy_grid"))

    print(f'El tiempo de ejecución fue el siguiente: {tiempo}')


from collections import defaultdict

def reverse_dict(file_name):
    file = open(file_name, 'r')
    lines = file.readlines()
    file.close()

    original = defaultdict(list)

    # split into common dictionary
    for line in lines:
        word, translations = line.strip().split(' - ')
        original[word].extend(translations.split(', '))

    reversed_dict = defaultdict(list)

    # reverse dictionary
    for key, values in original.items():
        for elem in values:
            reversed_dict[elem].append(key)

    # sort values
    for key in reversed_dict:
        reversed_dict[key].sort()

    output = open('output.txt', 'w')

    # write to new file
    for word in sorted(reversed_dict):
        output.write(word + ' - ' + ', '.join(reversed_dict[word]) + '\n')

    output.close()


import torch
from torch import nn
import torch.nn.functional as nnfunc

class Distillation(object):

    """
    Distillation loss class
    supports :param: method-->
        1. mse      : match yolo layer outputs
    """

    def __init__(self,method="mse"):
        if method=="mse":
            self.loss_fn = self.mse
        # elif method=="cfmse":
        #    self.loss_fn = self.cfmse
        # elif method=="cfmse2":
        #     self.loss_fn = self.cfmse2
        else:
            raise NotImplementedError

    def mse(self, predS, predT):
        """
        mse between predT & predS
        only works when Stu & Tea are same architecture
        """
        assert len(predT) == len(predS)
        dLoss = []
        for branchS, branchT in zip(predS, predT):
            dLoss.append(torch.mean((branchS - branchT)**2))
        dLoss = sum(dLoss)
        dLoss_items = torch.tensor((0.0, 0.0, 0.0, dLoss.item())).to(dLoss.device)
        return dLoss, dLoss_items.detach()


import myutils
from datetime import datetime
import sys
import os
import pickle
from keras.models import load_model
from gensim.models import Word2Vec, KeyedVectors
from keras.preprocessing import sequence
from sklearn.metrics import accuracy_score
from sklearn.metrics import precision_score
from sklearn.metrics import recall_score
from sklearn.metrics import f1_score
import tensorflow as tf
import numpy

#default mode / type of vulnerability
mode = "sql"

#get the vulnerability from the command line argument
if (len(sys.argv) > 1):
  mode = sys.argv[1]

model = load_model('model/LSTM_model_'+mode+'.h5',custom_objects={'f1_loss': myutils.f1_loss, 'f1':myutils.f1})

with open('data/' + mode + '_dataset_finaltest_X', 'rb') as fp:
  FinaltestX = pickle.load(fp)
with open('data/' + mode + '_dataset_finaltest_Y', 'rb') as fp:
  FinaltestY = pickle.load(fp)

now = datetime.now() # current date and time
nowformat = now.strftime("%H:%M")

#Prepare the data for the LSTM model

X_finaltest =  numpy.array(FinaltestX)
y_finaltest =  numpy.array(FinaltestY)

#in the original collection of data, the 0 and 1 were used the other way round, so now they are switched so that "1" means vulnerable and "0" means clean.

for i in range(len(y_finaltest)):
  if y_finaltest[i] == 0:
    y_finaltest[i] = 1
  else:
    y_finaltest[i] = 0

now = datetime.now() # current date and time
nowformat = now.strftime("%H:%M")

print(str(len(X_finaltest)) + " samples in the final test set.")

csum = 0
for y in y_finaltest:
  csum = csum+y

print("percentage of vulnerable samples: "  + str(int((csum / len(X_finaltest)) * 10000)/100) + "%")
print("absolute amount of vulnerable samples in test set: " + str(csum))

#padding sequences on the same length
max_length = 200
X_finaltest = sequence.pad_sequences(X_finaltest, maxlen=max_length)

yhat_classes = model.predict_classes(X_finaltest, verbose=0)
accuracy = accuracy_score(y_finaltest, yhat_classes)
precision = precision_score(y_finaltest, yhat_classes)
recall = recall_score(y_finaltest, yhat_classes)
F1Score = f1_score(y_finaltest, yhat_classes)

print("Accuracy: " + str(accuracy))
print("Precision: " + str(precision))
print("Recall: " + str(recall))
print('F1 score: %f' % F1Score)
print("\n")


from django.core.management.base import BaseCommand
from django_postgres_drop_index.utils import drop_table_indexes

class Command(BaseCommand):

    def add_arguments(self, parser):
        parser.add_argument('tablename', nargs='+')

    def handle(self, *args, **options):
        for tablename in options['tablename']:
            schemaname = 'public'
            if '.' in tablename:
                schemaname, tablename in tablename.split('.')
            drop_table_indexes(tablename, schemaname=schemaname)


from django.db import models

class Category(models.Model):
    class Meta:
        verbose_name = 'Category'
        verbose_name_plural = 'Categories'
    name = models.CharField(max_length=20)

    def __str__(self):
        return self.name

class Portfolio(models.Model):
    class Meta:
        verbose_name = 'Portfolio'
        verbose_name_plural = 'Portfolios'

    name = models.CharField(max_length=20)
    description = models.TextField(max_length=50)
    img = models.ImageField()
    category = models.OneToOneField(Category, on_delete=models.CASCADE)
    project_url = models.URLField(max_length=50)
    date = models.DateField()
    github_link = models.URLField(max_length=50)

    def __str__(self):
        return self.name

class Blog(models.Model):
    class Meta:
        verbose_name = 'Blog'
        verbose_name_plural = 'Blog'

    name = models.CharField(max_length=40)
    description = models.TextField(max_length=1000)
    description_next = models.TextField(max_length=1000)
    category = models.OneToOneField(Category, on_delete=models.CASCADE)
    img = models.ImageField()
    quote = models.CharField(max_length=50)
    date = models.DateField()

    def __str__(self):
        return self.name

class Partner(models.Model):
    name = models.CharField(max_length=50)
    about = models.CharField(max_length=100)
    img = models.ImageField()

    def __str__(self):
        return self.name


#!/usr/bin python
# -*- coding: UTF-8 -*-

import os

from sqlalchemy.exc import DBAPIError

from bottle import debug, route, template, run, static_file, get, post, request

from functions import getdatetime, get_companies, get_company_details
from config import PROJECT_PATH, DEBUG, MAIN_TITLE, CSS_FILES, JS_FILES

debug(DEBUG)

@route('/')
def index():
    page_title = "Kezdő oldal - {0}".format(MAIN_TITLE)
    css_files = CSS_FILES
    js_files = JS_FILES
    now = getdatetime()
    return template('index', page_title=page_title, css_files=css_files, js_files=js_files, now=now)

@route('/static/<filename:path>')
def static(filename):
    return static_file(filename, root='./static')

@route('/companies')
def companies_get():
    page_title = "Cégek - {0}".format(MAIN_TITLE)
    css_files = CSS_FILES
    js_files = JS_FILES
    now = getdatetime()
    companies = []
    errors = []

    prepost = {'nev': "", 'bankszamla': "", 'kapcsolattarto': "", 'operator': 'AND', 'rendezes': 'nev'}

    try:
        companies, _ = get_companies()
    except DBAPIError as e:
        if DEBUG:
            print(e.message)
            errors = ["Nem sikerült csatlakozni az adatbázishoz! :("]

    return template('companies', page_title=page_title, css_files=css_files, js_files=js_files, now=now, errors=errors, prepost=prepost, companies=companies)

@route('/companies', method='POST')
def companies_post():
    page_title = "Cégek - {0}".format(MAIN_TITLE)
    css_files = CSS_FILES
    js_files = JS_FILES
    now = getdatetime()
    companies = []
    errors = []

    nev = request.forms.get('nev')
    bankszamla = request.forms.get('bankszamla')
    kapcsolattarto = request.forms.get('kapcsolattarto')
    operator = request.forms.get('operator')
    rendezes = request.forms.get('rendezes')

    if DEBUG:
        print("POST -- nev: {0}, bankszamla: {1}, kapcsolattarto: {2}, operator: {3}, rendezes: {4}".format(nev, bankszamla, kapcsolattarto, operator, rendezes))

    try:
        companies, prepost = get_companies(nev=nev, bankszamla=bankszamla, kapcsolattarto=kapcsolattarto, operator=operator, rendezes=rendezes)
    except DBAPIError as e:
        if DEBUG:
            print(e.message)
            errors = ["Nem sikerült csatlakozni az adatbázishoz! :("]

    return template('companies', page_title=page_title, css_files=css_files, js_files=js_files, now=now, errors=errors, companies=companies, prepost=prepost)

@route('/company-details/<id:int>')
def companies(id):
    page_title = "Cég részletei - {0}".format(MAIN_TITLE)
    css_files = CSS_FILES
    js_files = JS_FILES
    now = getdatetime()
    company = []
    errors = []
    try:
        company = get_company_details(id)[0]
    except DBAPIError as e:
        if DEBUG:
            print(e.message)
            errors = ["Nem sikerült csatlakozni az adatbázishoz! :("]
    except IndexError:
        company = []
        errors = ["Nincs ilyen azonosítóval rendelkező cég az adatbázisban! :("]

    return template('company-details', page_title=page_title, css_files=css_files, js_files=js_files, now=now, errors=errors, company=company)

run(host='localhost', port='80', reloader=True)


from ideasdk.protocols import SocaPubSubProtocol, SubscriptionListener

from blinker import signal
from typing import Optional

class SocaPubSub(SocaPubSubProtocol):
    def __init__(self, topic: str, logger=None):
        self._signal = signal(topic)
        self._logger = logger

    def publish(self, sender: str, **kwargs):
        self._signal.send(sender, **kwargs)

    def subscribe(self, listener: SubscriptionListener, sender: Optional[str] = None):
        if self._logger:
            self._logger.debug(f'subscribe - topic: {self._signal.name}, listener: {listener}')

        if sender is None:
            self._signal.connect(listener)
        else:
            self._signal.connect(listener, sender=sender)

    def unsubscribe(self, listener: SubscriptionListener, sender: Optional[str] = None):
        if self._logger:
            self._logger.debug(f'unsubscribe - topic: {self._signal.name}, listener: {listener}')

        if sender is None:
            self._signal.disconnect(listener)
        else:
            self._signal.disconnect(listener, sender=sender)


import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt

df = pd.read_csv("masculinity.csv")
sns.countplot(x="how_masculine", data=df)
plt.show()

"""
Unnamed: 0               How old are you?
    0     Marion                             12
    1      Elroy                             16
    2        NaN  What is your favorite animal?
    3     Marion                            dog
    4      Elroy                            cat

Is it tidy?
No, because a single column contains different types of information.
"""

# adding a third variable with hue
# load dataset -> function in Seaborn and passing in the name of the dataset
tips = sns.load_dataset("tips")
tips.head()

sns.scatterplot(x="total_bill", y="tip", data="tips", hue="smoker", hue_order=["Yes", "No"])
# hue_order: takes in a list of values and will set the order of the values in the plot accordingly
# pallete: control the colors assigned to each value

hue_colors = {"Yes": "black", "No": "red"}
sns.scatterplot(x="total_bill",
                y="tip",
                data=tips,
                hue="smoker",
                palette=hue_colors,
                hue_order=["Yes", "No"])

# exercise
sns.scatterplot(x="absences", y="G3",
                data=student_data,
                hue="location",
                hue_order=["Rural", "Urban"])

# exercise II
palette_colors = {"Rural": "green", "Urban": "blue"}

# Create a count plot of school with location subgroups
sns.countplot(x="school", data=student_data, palette=palette_colors, hue="location")


import dis
from collections import defaultdict

import importlib
import pathlib
import os
import shutil

_extended_sys_path = []

def _common_root(path_list):
    path_list = [path.split(os.path.sep) for path in path_list]
    common_tokens = []
    for tokens in zip(*path_list):
        for i in range(len(tokens) - 1):
            if tokens[i] != tokens[1+i]: break
        else:
            common_tokens.append(tokens[0])
            continue
        break
    return os.path.sep.join(common_tokens)

def parse_dependency(py_path:str, root_dir):

    if not py_path.endswith(".py"): return []

    # parse entrypoint script for imported files
    with open(py_path, "r") as f:
        statements = f.read()

    instructions = dis.get_instructions(statements)
    imports = [__ for __ in instructions if 'IMPORT' in __.opname]

    grouped = defaultdict(list)
    for instr in imports:
        grouped[instr.opname].append(instr.argval)

    pkgs = grouped["IMPORT_NAME"]

    deps = []
    for pkg in pkgs:
        try:
            dep = importlib.util.find_spec(pkg).origin
            pathlib.Path(dep).relative_to(root_dir)
        except: continue
        if dep not in deps:
            deps.append(dep)
            deps += parse_dependency(dep, root_dir)
    return deps

def _backup_source_files_to(entrypoint:str, extra_backup_paths, target_dir:str):

    entrypoint = os.path.realpath(entrypoint)

    root_dir = _common_root(_extended_sys_path + [os.path.dirname(entrypoint)])

    deps = parse_dependency(entrypoint, root_dir)
    deps.append(entrypoint)

    for absdep in deps:
        reldep = str(pathlib.Path(absdep).relative_to(root_dir))
        tardep = os.path.join(target_dir, reldep)
        tardir = os.path.dirname(tardep)
        os.makedirs(tardir, exist_ok=True)
        shutil.copy2(absdep, tardep)

    for ebp in extra_backup_paths:
        try:
            relebp = str(pathlib.Path(ebp).relative_to(root_dir))
            tarebp = os.path.join(target_dir, relebp)
        except: continue

        if os.path.isdir(ebp):
            shutil.copytree(ebp, tarebp)
        elif os.path.isfile(ebp):
            tardir = os.path.dirname(tarebp)
            os.makedirs(tardir, exist_ok=True)
            shutil.copy2(ebp, tarebp)