#!/usr/bin/python3

import datetime
import stat
import os
from pathlib import Path
import secrets
import logging
import ipaddress

from cryptography import x509
from cryptography.hazmat.primitives import serialization as crypto_serialization
from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.backends import default_backend as crypto_default_backend
from cryptography.x509.oid import NameOID
import nacl
from nacl import secret
import hashlib
import base64

from gi.repository import GLib, GObject

import util
import prefs

day = datetime.timedelta(1, 0, 0)
EXPIRE_TIME = 30 * day

DEFAULT_GROUP_CODE = "Warpinator"
KEYFILE_GROUP_NAME = "warpinator"
KEYFILE_CODE_KEY = "code"
KEYFILE_UUID_KEY = "connect_id"
CONFIG_FILE_NAME = ".group"

CONFIG_FOLDER = os.path.join(GLib.get_user_config_dir(), "warpinator")

singleton = None

def get_singleton():
    global singleton

    if singleton == None:
        singleton = AuthManager()

    return singleton

class AuthManager(GObject.Object):
    __gsignals__ = {
        'group-code-changed': (GObject.SignalFlags.RUN_LAST, None, ())
    }

    def __init__(self):
        GObject.Object.__init__(self)
        self.hostname = util.get_hostname()
        self.ident = ""
        self.ip_info = None
        self.port = None
        self.code = ""

        self.private_key = None
        self.server_cert = None

        self.remote_certs = {}

        os.makedirs(CONFIG_FOLDER, exist_ok=True)
        self.path = Path(os.path.join(CONFIG_FOLDER, CONFIG_FILE_NAME))

    def update(self, ip_info, port):
        self.ip_info = ip_info;
        self.port = port

        self._load_keyfile()

        self._read_ident()
        self._read_group_code()

        self._make_key_cert_pair()

    def get_ident(self):
        return self.ident

    def get_group_code(self):
        return self.code;

    def update_group_code(self, code):
        if code == self.code:
            return

        self.keyfile.set_string(KEYFILE_GROUP_NAME, KEYFILE_CODE_KEY, code)
        self._save_keyfile()

        self.emit("group-code-changed")

    def get_server_creds(self):
        return (self.server_private_key, self.server_pub_key)

    def get_cached_cert(self, hostname, ip_info):
        try:
            return self.remote_certs["%s.%s" % (hostname, ip_info.ip4_address)]
        except KeyError:
            return None

    def process_remote_cert(self, hostname, ip_info, server_data):
        if server_data == None:
            return False
        decoded = base64.decodebytes(server_data)

        hasher = hashlib.sha256()
        hasher.update(bytes(self.code, "utf-8"))
        key = hasher.digest()
        decoder = secret.SecretBox(key)

        try:
            cert = decoder.decrypt(decoded)
        except nacl.exceptions.CryptoError as e:
            print(e)
            cert = None

        if cert:
            self.remote_certs["%s.%s" % (hostname, ip_info.ip4_address)] = cert
            return True
        else:
            return False

    def get_encoded_local_cert(self):
        hasher = hashlib.sha256()
        hasher.update(bytes(self.code, "utf-8"))
        key = hasher.digest()

        encoder = secret.SecretBox(key)

        encrypted = encoder.encrypt(self.server_pub_key)
        encoded = base64.encodebytes(encrypted)
        return encoded

# Internals
    def _load_keyfile(self):
        self.keyfile = GLib.KeyFile()

        try:
            self.keyfile.load_from_file(self.path.as_posix(), GLib.KeyFileFlags.NONE)
        except GLib.Error as e:
            if e.code == GLib.FileError.NOENT:
                logging.debug("Auth: No group code file, making one.")
                self.path.touch()
            else:
                logging.debug("Auth: Could not load existing keyfile (%s): %s" %(CONFIG_FOLDER, e.message))
                self.path.unlink()
                self.path.touch()

    def _save_keyfile(self):
        keyfile_bytes = bytes(self.keyfile.to_data()[0], "utf-8")

        try:
            self.path.unlink()
        except OSError:
            pass

        flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
        mode = stat.S_IRUSR | stat.S_IWUSR
        umask = 0o777 ^ mode  # Prevents always downgrading umask to 0.

        umask_original = os.umask(umask)

        try:
            fdesc = os.open(self.path, flags, mode)
        finally:
            os.umask(umask_original)

        with os.fdopen(fdesc, 'wb') as f:
            f.write(keyfile_bytes)

    def _read_ident(self):
        gen_new = False

        try:
            self.ident = self.keyfile.get_string(KEYFILE_GROUP_NAME, KEYFILE_UUID_KEY)
        except GLib.Error as e:
            if e.code not in (GLib.KeyFileError.KEY_NOT_FOUND, GLib.KeyFileError.GROUP_NOT_FOUND):
                logging.critical("Could not read uuid (ident) from settings file: %s" % e.message)

            gen_new = True

        if len(self.ident.split("-")) == 5:
            gen_new = True

        if gen_new:
            # Max 'instance' length is 63.
            # https://datatracker.ietf.org/doc/html/rfc6763#section-7.2
            self.ident = "%s-%s" % (self.hostname.upper()[:42], secrets.token_hex(10).upper())
            self.keyfile.set_string(KEYFILE_GROUP_NAME, KEYFILE_UUID_KEY, self.ident)
            self._save_keyfile()

    def _read_group_code(self):
        reset = False
        code = None

        try:
            code = self.keyfile.get_string(KEYFILE_GROUP_NAME, KEYFILE_CODE_KEY)
        except GLib.Error as e:
            if e.code not in (GLib.KeyFileError.KEY_NOT_FOUND, GLib.KeyFileError.GROUP_NOT_FOUND):
                logging.warn("Could not read group code from settings file (%s): %s" % (CONFIG_FOLDER, e.message))

        if code == None or code == "":
            self.code = DEFAULT_GROUP_CODE
            self._save_keyfile()
            return

        if len(code) < 8:
            logging.warn("Group Code is short, consider something longer than 8 characters.")

        self.code = code

    def _make_key_cert_pair(self):
        logging.debug("Auth: Creating server credentials")

        private_key = rsa.generate_private_key(
            backend=crypto_default_backend(),
            public_exponent=65537,
            key_size=2048
        )

        public_key = private_key.public_key()

        builder = x509.CertificateBuilder()
        builder = builder.subject_name(x509.Name([
            x509.NameAttribute(NameOID.COMMON_NAME, self.hostname),
        ]))
        builder = builder.issuer_name(x509.Name([
            x509.NameAttribute(NameOID.COMMON_NAME, self.hostname),
        ]))
        builder = builder.not_valid_before(datetime.datetime.today() - day)
        builder = builder.not_valid_after(datetime.datetime.today() + EXPIRE_TIME)
        builder = builder.serial_number(x509.random_serial_number())
        builder = builder.public_key(public_key)

        alt_names = []

        if self.ip_info.ip4_address != None:
            alt_names.append(x509.IPAddress(ipaddress.IPv4Address(self.ip_info.ip4_address)))

        builder = builder.add_extension(x509.SubjectAlternativeName(alt_names), critical=True)

        certificate = builder.sign(
            private_key=private_key, algorithm=hashes.SHA256(),
            backend=crypto_default_backend()
        )

        ser_private_key = private_key.private_bytes(
            crypto_serialization.Encoding.PEM,
            crypto_serialization.PrivateFormat.PKCS8,
            crypto_serialization.NoEncryption())

        ser_public_key = certificate.public_bytes(
            crypto_serialization.Encoding.PEM
        )

        self.server_pub_key = ser_public_key
        self.server_private_key = ser_private_key


# -*- coding: utf-8 -*-
'''
Created on 2020.

@author: aon
'''
from tools.DC import DCC, well, toWell
from api.formTools import style, _div, pyramid, pyramidInit, _mainPage, _field
from api.classPage import Page

# *** *** ***

class t__list(Page):

    def __init__(self, dcUK):
        self.noCaching = True
        self.jsCssUrl = [
            'jsv?api/react/_pyramid.js',
            'jsv?api/pages/t__list/t__list.js',
            'jsv?api/pages/t_lesson/t_lesson.js',
            'jsv?api/react/view.css']

        self.cats = {
            'Курсы': DCC(condition='True', sort='docNo', reverse=True,
                fixedSubCats=[
                    DCC(subCat='По\xa0названию', condition='True', sort='trackName'),
                ]
            ),
            'По\xa0категории': DCC(condition='True', sort='trackName', subCat="d.cat or '-'"),
            'По\xa0кураторам': DCC(condition='True', sort='trackName', subCat="d.curator or '-'"),
        }

        super().__init__(dcUK)

    # *** *** ***

    def page(self, dcUK):
        self.title = f"Курс {dcUK.dbAlias.rpartition('_')[2]}"

        dcVP = DCC(cats=self.cats, form='t_lesson', dbAlias=dcUK.dbAlias)

        dcVP['checkBox'] = _div(
            **style(width=380, verticalAlign='middle'), children=
            [
                _field('active', 'chb3', ['активные', 'ожидание'], **style(letterSpacing=1))
            ]
        )

        view = self.getViewObject('ViewLessons', dcVP, 'ViewLessons')

        return _mainPage(className='page51', children=[
            pyramid(),

            _div(**style(maxWidth=1000, margin='auto'), className='cellbg-green', children=
                view.make()  # вид с кнопками слева  и тулбаром
            )
        ])

    def queryOpen(self, dcUK):
        dcUK.doc._page_ = '1'
        pyramidInit(dcUK)

# *** *** ***


# Uses python3
import sys
from collections import namedtuple

Segment = namedtuple('Segment', 'start end')

def optimal_points(segments):
    points = []
    segments.sort(key=lambda x: x[1])
    n = len(segments)
    while n>0:
        s = segments[0]
        points.append(s.end)
        segments = [Segment(x[0], x[1]) for x in segments if x[0] > s.end]
        n = len(segments)
    return points

if __name__ == '__main__':
    input = sys.stdin.read()
    n, *data = map(int, input.split())
    segments = list(map(lambda x: Segment(x[0], x[1]), zip(data[::2], data[1::2])))
    points = optimal_points(segments)
    print(len(points))
    print(*points)


import nltk
import re
import time
from sklearn.datasets import fetch_20newsgroups
from sklearn.decomposition import PCA
from sklearn.feature_extraction.text import TfidfVectorizer, TfidfTransformer
from string import digits
from sklearn.cluster import KMeans, Birch
from sklearn.cluster import DBSCAN
import matplotlib.pyplot as plt

# 去除文本里面的符号
def pre_process(data):
    text_new = []
    for text in data:
        remove_digits = str.maketrans('', '', digits)
        res = text.translate(remove_digits)
        result = res.replace('_', '')
        result = result.replace('*', '')
        result = result.replace('/', '')
        result = result.replace('\\', '')
        result = result.replace('{', '')
        result = result.replace('}', '')
        result = result.replace('-', '')
        result = result.replace('(', '')
        result = result.replace(')', '')
        text_new.append(result)
    return text_new

# 载入nltk的英文停用词作为停用词变量
stopwords = nltk.corpus.stopwords.words('english')

# 载入nltk的SnowballStemmer作为词干化变量
stemmer = nltk.SnowballStemmer("english")

# 分句加分词，分词后检查单词正确性并还原单词原型
def tokenize_and_stem(text):
    # 首先分句，接着分词，而标点也会作为词例存在
    tokens = [word for sent in nltk.sent_tokenize(text) for word in nltk.word_tokenize(sent)]
    filtered_tokens = ''
    # 过滤所有不含字母的词例（例如：数字、纯标点）
    for token in tokens:
        if re.search('[a-zA-Z]', token):
            if token not in stopwords:
                filtered_tokens += ' ' + stemmer.stem(token)
    return filtered_tokens

# 只分句分词，并都转换为小写
def tokenize_only(text):
    # 首先分句，接着分词，而标点也会作为词例存在
    tokens = [word.lower() for sent in nltk.sent_tokenize(text) for word in nltk.word_tokenize(sent)]
    filtered_tokens = []
    # 过滤所有不含字母的词例（例如：数字、纯标点）
    for token in tokens:
        if re.search('[a-zA-Z]', token):
            filtered_tokens.append(token)
    return filtered_tokens

categories = ['alt.atheism', 'soc.religion.christian', 'comp.graphics', 'sci.med']
twenty_train = fetch_20newsgroups(subset='train', categories=categories, shuffle=True, random_state=42)

count_vect  = TfidfVectorizer(min_df=3)
X_train_counts = count_vect.fit_transform(twenty_train.data)

tfidf_transformer = TfidfTransformer()
X_train_tfidf = tfidf_transformer.fit_transform(X_train_counts)

pca = PCA(n_components=2)  # 进行PCA降维
newdata = pca.fit_transform(X_train_tfidf.toarray())
plt.scatter(newdata[:, 0], newdata[:, 1])
plt.show()

# 进行kmeans聚类
num_clusters = 5
km = KMeans(n_clusters=num_clusters)
start = time.time()
result = km.fit_predict(newdata)
end = time.time()
print("k_means运行的时间是：", end-start)
plt.scatter(newdata[:, 0], newdata[:, 1], c=result)
plt.show()

# 进行dbscan聚类
start = time.time()
db = DBSCAN(eps=0.05, min_samples=30).fit_predict(newdata)
end = time.time()
print("DBscan运行的时间是：", end-start)
plt.scatter(newdata[:, 0], newdata[:, 1], c=db)
plt.show()

# 进行birch聚类
start = time.time()
result_birch = Birch(n_clusters=5).fit_predict(newdata)
end = time.time()
print("birch运行的时间是：", end-start)
plt.scatter(newdata[:, 0], newdata[:, 1], c=result_birch)
plt.show()


#%%
import seaborn as sns
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import copy
from collections import deque
from pandas.core.frame import DataFrame
from sklearn import metrics
import scipy.stats as st
from collections import defaultdict
import math
from scipy.interpolate import interp1d
from scipy.interpolate import PchipInterpolator
#%%
def df_get_pdf(df,X,Y,X_len,delta=None,CDF_Monotonic=True):
    def feature_cdf(df,X,Y,ascending=True):
        def zero():
            return 0
        # df sort
        arr_len = df.shape[0]
        feat_N = df[Y].unique()

        # Y_arrary
        Y_arr = np.array(df[Y])
        PY_arr = np.zeros((arr_len,feat_N.shape[0]),dtype=float)
        columns = ["%s_cdf(%s=%s)" %("As" if ascending==True else "Ds",Y,v) for i,v in enumerate(feat_N)]

        tmp_N = defaultdict(zero)
        for ind_,val_ in enumerate(Y_arr):
            for fi_,fv_ in enumerate(feat_N):
                if val_ == fv_:
                    tmp_N[fv_] += 1
                    PY_arr[ind_,fi_] = tmp_N[fv_]
                else:
                    PY_arr[ind_,fi_] = tmp_N[fv_]

        for fi_,fv_ in enumerate(feat_N):
            PY_arr[:,fi_] = PY_arr[:,fi_] / tmp_N[fv_]

        df_cdf = pd.concat([df[X],
                            pd.DataFrame(PY_arr,columns=columns)],axis=1)\
                    .groupby(by=[X]).max()
        return df_cdf.sort_index()

    def get_interfunc(df,Y):
        X_Y = np.zeros((df.shape[0] + 2,2))
        X_Y[1:-1,0] = df.index
        X_Y[1:-1,1] = df[Y]
        X_Y[0] = 2 * X_Y[1] - X_Y[2]
        X_Y[-1] = 2 * X_Y[-2] - X_Y[-3]
        X_Y[0,1] = 0
        X_Y[-1,1] = 1

        if CDF_Monotonic == True:
            func_ = PchipInterpolator(X_Y[:,0], X_Y[:,1])
        else:
            func_ = interp1d(X_Y[:,0], X_Y[:,1], kind = 'quadratic')
        return func_, X_Y[-2,0] - X_Y[-3,0]

    df_sort = df.sort_values(by=[X]).reset_index(drop=True)
    feat_X = df_sort[X].to_numpy()  #格式轉換

    interval_n = math.ceil((feat_X.max()-feat_X.min())/X_len)
    feat_Xmin = feat_X.min()
    for i in range(interval_n,-1,-1):
        feat_X[(feat_Xmin + (i)*X_len <= feat_X) &\
               (feat_X < feat_Xmin + (i+1)*X_len) ] = feat_Xmin + (i+1)*X_len

    feat_cdf = feature_cdf(df_sort, X=X, Y=Y)
    cols = feat_cdf.columns

    if delta == None:
        delta = X_len/2

    X_delta1 = feat_X - X_len/2 + delta
    X_delta2 = feat_X - X_len/2 - delta
    for col in cols:
        XY_func,delta_tmp = get_interfunc(feat_cdf,col)
        Y_delta1 = XY_func(X_delta1)
        Y_delta2 = XY_func(X_delta2)
        df_sort['pdf' + col[6:]] = (Y_delta2 - Y_delta1 ) /(X_delta2 - X_delta1)

    df_sort[X] = feat_X - X_len/2

    return df_sort

#%%
if __name__ == "__main__":
    #data prepare
    mu, sigma = 10, 5
    s1 = st.norm(mu, sigma).rvs(100000)
    s2 = st.norm(mu-4, sigma+4).rvs(100000)

    df_1 = pd.DataFrame(s1,columns=["feat"]  )
    df_1['Y'] = 0

    df_2 = pd.DataFrame(s2,columns=["feat"]  )
    df_2['Y'] = 1
    df = pd.concat([df_1,df_2],axis=0).reset_index(drop=True)

#%%
X_len = 1
delta = .5
X = 'feat'
Y = 'Y'
bins = math.ceil((df[X].max()-df[X].min())/X_len)-2

# df_get_pdf
df_pdf = df_get_pdf(df, X=X, Y=Y, X_len=X_len, delta=delta,CDF_Monotonic=True)
df_pdf['pdf_diff'] = df_pdf['pdf(Y=1)'] - df_pdf['pdf(Y=0)']
df[X].min()
plt.figure(figsize=(15,10),dpi=100,linewidth = 2)
plt.hist(df.loc[df[Y]==0,X],bins=bins,density=True)
plt.hist(df.loc[df[Y]==1,X],bins=bins,density=True)
plt.plot(df_pdf[X],df_pdf['pdf(Y=1)'],'o-',color = 'r', label="pdf(Y=1)")
plt.plot(df_pdf[X],df_pdf['pdf(Y=0)'],'o-',color = 'g', label="pdf(Y=0)")
#plt.plot(df_pdf['feat'],df_pdf['pdf_diff'],'-',color = 'b', label="pdf_diff")

plt.xticks(fontsize=20)
plt.yticks(fontsize=20)

# 標示x,y軸(labelpad代表與圖片的距離)
plt.xlabel("feat", fontsize=30, labelpad = 15)
plt.ylabel("pdf", fontsize=30, labelpad = 20)

# 顯示出線條標記位置
plt.legend(loc = "best", fontsize=20)
plt.show()


#!/usr/bin/python
import requests
import json

base_url='http://localhost:5000'

book_list = [
{"title" : "alfajor",
"publisher" : "planeta",
"editionYear" : 2010,
"editionCountry" : "argentina",
"price" : 100,
"isbn" : "2128629716",
"gender" : "fisica",
"reputationValue" : 6,
"loanType" : "REMOTE"},
{"title" : "harry potter",
"publisher" : "planeta",
"editionYear" : 2010,
"editionCountry" : "argentina",
"price" : 200,
"isbn" : "3029855590",
"gender" : "fisica",
"reputationValue" :  8,
"loanType" : "LOCAL"},
{"title" : "pacman trumpeter",
"publisher" : "emece",
"editionYear" : 2010,
"editionCountry" : "argentina",
"price" : 10.50,
"isbn" : "2332932199",
"gender" : "terror",
"reputationValue" : 4,
"loanType" : "REMOTE"},
{"title" : "gringos amigos",
"publisher" : "golondrina",
"editionYear" : 2010,
"editionCountry" : "argentina",
"price" : 10.50,
"isbn" : "1621017673",
"gender" : "terror",
"reputationValue" : 6,
"loanType" : "LOCAL"},
{"title" : "gefatura de bebe gloglo",
"publisher" : "salamndra",
"editionYear" : 2010,
"editionCountry" : "argentina",
"price" : 10.50,
"isbn" : "2649520825",
"gender" : "legislatura",
"reputationValue" : 3,
"loanType" : "REMOTE"},
]

book_id_list = []
for book in book_list:
        response = requests.post(base_url + "/books",
                   data = json.dumps(book),
                   headers={'Content-Type': 'application/json'})
        book_id_list.append(response.json()['id'])

author_list = [
{"firstName" : "pablo",
"lastName" : "professi",
"nationality" : "argentino"},
{"firstName" : "javier",
"lastName" : "anselmi",
"nationality" : "argentino"},
{"firstName" : "kuasimodo",
"lastName" : "borbonete",
"nationality" : "frances"},
{"firstName" : "quintaro",
"lastName" : "grindustre",
"nationality" : "holanda"},
{"firstName" : "gonzo",
"lastName" : "bonneli",
"nationality" : "hungria"},
{"firstName" : "ratimi",
"lastName" : "tripipio",
"nationality" : "kasajstan"},
]

author_id_list = []
for author in author_list:
        response = requests.post(base_url + "/authors",
                   data = json.dumps(author),
                   headers={'Content-Type': 'application/json'})
        author_id_list.append(response.json()['id'])

requests.post(base_url + "/books/" + book_id_list[0] + "/authors/" + author_id_list[0], data = json.dumps({}),headers={'Content-Type': 'application/json'})
requests.post(base_url + "/books/" + book_id_list[0] + "/authors/" + author_id_list[1], data = json.dumps({}),headers={'Content-Type': 'application/json'})
requests.post(base_url + "/books/" + book_id_list[1] + "/authors/" + author_id_list[2], data = json.dumps({}),headers={'Content-Type': 'application/json'})
requests.post(base_url + "/books/" + book_id_list[1] + "/authors/" + author_id_list[3], data = json.dumps({}),headers={'Content-Type': 'application/json'})
requests.post(base_url + "/books/" + book_id_list[2] + "/authors/" + author_id_list[4], data = json.dumps({}),headers={'Content-Type': 'application/json'})
requests.post(base_url + "/books/" + book_id_list[2] + "/authors/" + author_id_list[2], data = json.dumps({}),headers={'Content-Type': 'application/json'})
requests.post(base_url + "/books/" + book_id_list[2] + "/authors/" + author_id_list[0], data = json.dumps({}),headers={'Content-Type': 'application/json'})

#response = requests.post(base_url + "/members",
#            data = json.dumps({'firstName' : 'pabloide',
#                               'lastName' : 'profelmi',
#                               'dni' :  '35666666',
#                               'nationality' :
#                               'Argentina','cuil' :
#                               '20356666662','phone' :
#                               '0540111557781234',
#                               'email' : 'asdasd@gmail.com',
#                               'zipCode' : '1602',
#                               'city' : 'vicente lopez',
#                               'state' : 'buenos aires',
#                               'enabled' : 'True',
#                               'reputation' : '7'}),
#            headers={'Content-Type': 'application/json'})
#id_member_A_created = response.json()['id']
#response = requests.post(base_url + "/members",
#            data = json.dumps({'firstName' : 'javioide',
#                               'lastName' : 'profelmi',
#                               'dni' :  '357777777',
#                               'nationality' : 'Argentina',
#                               'cuil' : '203577777772',
#                               'phone' : '0540111557666666',
#                               'email' : 'wwwwwwwwwwww@gmail.com',
#                               'zipCode' : '1632',
#                               'city' : 'vicente lopez',
#                               'state' : 'buenos aires',
#                               'enabled' : 'True',
#                               'reputation' : '8'}),
#            headers={'Content-Type': 'application/json'})
#id_member_B_created = response.json()['id']

#response = requests.post(base_url + "/samples",
#            data = json.dumps({"bookId": "1",
#                               "barCode":"1234567890",
#                               "acquisitionDate": "2016-10-06",
#                               "discardDate": "2016-10-13"}),
#            headers={'Content-Type': 'application/json'})
#id_sample_A_created = response.json()['id']
#response = requests.post(base_url + "/samples",
#            data = json.dumps({"bookId": "1" ,
#                               "barCode":"000000000",
#                               "acquisitionDate": "2016-10-23"}),
#            headers={'Content-Type': 'application/json'})
#id_sample_B_created = response.json()['id']
#response = requests.post(base_url + "/samples",
#            data = json.dumps({"bookId": "1",
#                               "barCode":"1111111111",
#                               "acquisitionDate": "2016-10-23"}),
#            headers={'Content-Type': 'application/json'})
#id_sample_C_created = response.json()['id']

#response = requests.post(base_url + "/loans",
#            data = json.dumps({'agreedReturnDate' : '2016-10-24',
#                               'withdrawDate' : '2016-10-24',
#                               'loanType': 'REMOTE',
#                               'sampleId': "2",
#                               'memberId': "1"}),
#            headers={'Content-Type': 'application/json'})
#print(response.text)
#response = requests.post(base_url + "/loans",
#            data = json.dumps({'agreedReturnDate' :  "2016-10-19",
#                               'withdrawDate' : "2016-10-24",
#                               "loanType": "REMOTE",
#                               'sampleId': "3",
#                               'memberId': "1"}),
#            headers={'Content-Type': 'application/json'})
#response = requests.post(base_url + "/loans",
#            data = json.dumps({'agreedReturnDate' : '2016-10-24',
#                               'returnDate': None,
#                               'withdrawDate' : '2016-10-22',
#                               'comment' : 'soy de atlanta y q?',
#                               'loanType': 'LOCAL',
#                               'sampleId': id_sample_B_created,
#                               'memberId': id_member_A_created}),
#            headers={'Content-Type': 'application/json'})
#response = requests.post(base_url + "/loans",
#            data = json.dumps({'agreedReturnDate' :  "2016-10-24",
#                               'returnDate': "2016-10-22",
#                               'withdrawDate' : "2016-10-04",
#                               'comment' : "gol gol gol",
#                               "loanType": "LOCAL",
#                               'sampleId': id_sample_B_created,
#                               'memberId': id_member_B_created}),
#            headers={'Content-Type': 'application/json'})


"""
Spatial statistics.
"""

__author__ = "Steven Kearnes"
__copyright__ = "Copyright 2014, Stanford University"
__license__ = "3-clause BSD"

import numpy as np
from scipy.spatial.distance import cdist, is_valid_dm

def distance(a, b):
    """
    Calculate pairwise distances between examples in two datasets.

    Parameters
    ----------
    a, b : ndarray
        Descriptor datasets with examples on the first axis and descriptors
        on the second axis.
    """
    return cdist(a, b)

def spread(d, t):
    """
    Calculate the spread between two sets of compounds.

    Given a matrix containing distances between two sets of compounds, A
    and B, calculate the fraction of compounds in set A that are closer
    than t to any compound in set B.

    If the distance matrix is symmetric and has zero diagonals, the
    diagonal values are ignored.

    Parameters
    ----------
    d : ndarray
        Distance matrix with compounds from set A on first axis.
    t : float
        Distance threshold.
    """

    # test for symmetric distance matrix
    # diagonal values should be ignored
    if is_valid_dm(d):
        d = np.copy(d)
        d[np.diag_indices_from(d)] = np.inf
    s = np.mean(np.any(d < t, axis=1))
    return s

def sum_of_spreads(d, coeff, min_t=0, max_t=3, step=None, diff=None):
    """
    Calculate the sum of spreads (or spread differences) across a range of
    distance thresholds.

    Parameters
    ----------
    d : ndarray
        Distance matrix.
    coeff : float
        Coefficient used to rescale distance thresholds.
    min_t : float, optional (default 0)
        Minimum distance threshold (before rescaling).
    max_t : float, optional (default 3)
        Maximum distance threshold (before rescaling).
    step : float, optional
        Step size for determining values to sample between min_t and max_t.
        If not provided, defaults to max_t / 500.
    diff : ndarray, optional
        Distance matrix. If provided, the spread will be calculated for this
        distance matrix and subtracted from the spread of d at each distance
        threshold.
    """
    if step is None:
        step = max_t / 500.
    n_steps = int((max_t - min_t) / step)
    thresholds = coeff * np.linspace(min_t, max_t, n_steps)
    if diff is not None:
        ss = np.sum([spread(d, t) - spread(diff, t)
                     for t in thresholds])
    else:
        ss = np.sum([spread(d, t) for t in thresholds])
    return ss


from django import forms
from django.db import models
from django.contrib.auth.models import User
from django.contrib.auth.models import AbstractUser
from django.contrib.auth.forms import UserCreationForm
from .models import Post,RoomCategory
from django.forms.widgets import DateInput

class HostelUpdateForm(forms.Modelform):
    model = Post
    fields = [
        "hostel_name",
        "address",
        "area",
        "city",
        "contact",
        "rating",
        "breakfast",
        "lunch",
        "dinner",
        "transportation",
        "cctv",
        "fridge",
        "washing_machine",
        "geyser",
        "ac",
        "author"
    ]


import scrapy
from machineMax.items import MachinemaxItem

class GuardianSpider(scrapy.Spider):
    """
        Spider class

        Attributes
        ----------
            name : str
                Name of the spider.
            allowed_domains : list
                This is a list of allowed domains to scrap.
            start_urls : list
                This is a list of URLs to crawl.
    """
    name = "theguardian"
    allowed_domains = ['www.theguardian.com']
    start_urls = ['https://www.theguardian.com/']

    def parse(self, response):
        """
            This method processes the response and returns scrapped data following more URLs

            Parameters
            ----------
            response

            Returns
            -------
            yields scrapy.Request with call back to parsearticle()
        """
        # The path to article url as found using inspect element
        article_urls_xpath = '//*[contains(@class,"fc-item__link")]//@href'
        for article_url in response.xpath(article_urls_xpath).extract():
            yield scrapy.Request(
                url=article_url,
                callback=self.parsearticle
            )

    def parsearticle(self, response):
        """
            This method parses the crawled data

            Parameters
            ----------
            response

            Returns
            -------
            MachinemaxItem
                Yields parsed article
        """
        # Test if the article is already crawled
        if ('cached' in response.flags):
            return

        # Various paths retrieved by inspecting various articles
        headline_xpath = '//*[contains(@data-gu-name,"headline")]//text()'
        content_xpath = '//*[contains(@data-gu-name,"body")]//p//text()'
        author_xpath = '//*[contains(@rel,"author")]//*/text()'
        published_at_xpath = '//*[contains(@class,"content__dateline-wpd")]//@datetime'

        # Initialise response type
        item = MachinemaxItem()

        item['author'] = response.xpath(author_xpath).extract()
        item['headline'] = response.xpath(headline_xpath).extract_first()
        item['content'] = ''.join(response.xpath(content_xpath).extract())
        item['url'] = response.request.url
        item['published_at'] = response.xpath(published_at_xpath).extract_first()

        yield item


import common
from MenuTree import *
from codegen import MenuHandler

class PerlCodeGenerator:
    def get_properties_code(self, obj):
        return []

    def get_init_code(self, obj):
        prop = obj.properties
        plgen = common.code_writers['perl']
        out = []
        append = out.append
        menus = obj.properties['menubar']
        ids = []
        # We need to keep track of tmpnames used.
        tmpsused = {}

        def append_items(menu, items):
            for item in items:
                if item.name == '---': # item is a separator
                    append('%s->AppendSeparator();\n' % menu)
                    continue
                name, val = plgen.generate_code_id(None, item.id)
                if not name and ( not val or val == '-1'):
                    id = 'Wx::NewId()'
                else:
                    if name: ids.append(name)
                    id = val

                if item.children:
                    if item.name:
                        name = item.name
                    else:
                        name = '%s_sub' % menu
                        if not tmpsused.has_key(name):
                            tmpsused[name] = 1
                            append('my %s;\n' % name)

                    append('%s = Wx::Menu->new();\n' % name)
                    append_items(name, item.children)
                    append('%s->Append(%s, %s, %s, %s);\n' %
                           (menu, id, plgen.quote_str(item.label),
                            name, plgen.quote_str(item.help_str)))
                else:
                    item_type = 0
                    if item.checkable == '1':
                        item_type = 1
                    elif item.radio == '1':
                        item_type = 2

                    if item.name: itemname = '$self->{%s} = ' % item.name
                    else: itemname = ''

                    if item_type:
                        append('%s%s->Append(%s, %s, %s, %s);\n' %
                               (itemname, menu, id, plgen.quote_str(item.label),
                                plgen.quote_str(item.help_str), item_type))
                    else:

                        append('%s%s->Append(%s, %s, %s);\n' %
                               (itemname, menu, id, plgen.quote_str(item.label),
                                plgen.quote_str(item.help_str)))
        #print 'menus = %s' % menus

        if obj.is_toplevel: obj_name = '$self'
        else: obj_name = '$self->{%s}' % obj.name

        append('my $wxglade_tmp_menu;\n') # NOTE below name =
        for m in menus:
            menu = m.root
            if menu.name: name = '$self->{%s}' % menu.name
            else: name = '$wxglade_tmp_menu'
            append('%s = Wx::Menu->new();\n' % name)
            if menu.children:
                append_items(name, menu.children)
            append('%s->Append(%s, %s);\n' %
                   (obj_name, name, plgen.quote_str(menu.label)))

        return ids + out

    def get_code(self, obj):
        """\
        function that generates Perl code for the menubar of a wxFrame.
        """
        klass = obj.base;
        if klass != obj.klass : klass = obj.klass;
        else: klass = klass.replace('wx','Wx::',1);

        plgen = common.code_writers['perl']
        init = [ '\n\n', '# Menu Bar\n\n', '$self->{%s} = %s->new();\n' %
                 (obj.name, klass) ]
##                  '$self->SetMenuBar($self->{%s});\n' % obj.name ]
        init.extend(self.get_init_code(obj))
        init.append('$self->SetMenuBar($self->{%s});\n' % obj.name)
        init.append('\n# Menu Bar end\n\n')
        return init, [], []

    # 2004-12-05
    def get_events(self, obj):
        pygen = common.code_writers['perl']
        cn = pygen.cn
        out = []

        #print 'get_events', obj.properties['menubar']

        def do_get(item):
            ret = []
            if item.name:
                #val = '#self.%s' % item.name # see py_codegen.py, ~480
                val = item.name
            else:
                name, val = pygen.generate_code_id(None, item.id)
                if not val: val = '-1' # but this is wrong anyway...
            if item.handler:
                ret.append((val, 'EVT_MENU', item.handler))
            if item.children:
                for c in item.children:
                    ret.extend(do_get(c))
            return ret

        for menu in obj.properties['menubar']:
            out.extend(do_get(menu.root))
        return out

# end of class PerlCodeGenerator

def initialize():
    common.class_names['EditMenuBar'] = 'wxMenuBar'
    common.toplevels['EditMenuBar'] = 1

    plgen = common.code_writers.get('perl')
    if plgen:
        plgen.add_widget_handler('wxMenuBar', PerlCodeGenerator())
        plgen.add_property_handler('menus', MenuHandler)


import os
import pdb
import math
import time

from collections import OrderedDict

import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F

from darktorch.utils import calculate_iou, one_hot_encode, write_tensor

class RegionLossV3(nn.Module):
    def __init__(self, priors, threshold=0.5, n_classes=20):
        super().__init__()
        self.priors = priors
        self.threshold = threshold
        self.n_classes = n_classes
        self.obj_scale = 5
        self.nobj_scale = 1

        self.biases = None
        self.seen = 0

    def forward(self, output, target):
        tmp_target = target.cpu()
        tmp_target[tmp_target == -1] = 0
        write_tensor(tmp_target, 'target.binlog', False, False)

        K = 5 + self.n_classes
        B, Z, H, W = output.shape
        N = Z // K
        T = target.shape[1]
        scale_factor = 32

        output = output.permute((0, 2, 3, 1)).view(B, H, W, N, K).contiguous()
        deltas = torch.zeros(output.shape, device=output.device)

        deltas[:, :, :, :, 4] = self.nobj_scale * (0 - output[:, :, :, :, 4])

        # avg_anyobj = output[:, :, :, :, 4].sum() / (H * W * N * B)

        # if self.biases is None:
        #     self.biases = self.calculate_biases(B, H, W, N).to(output.device)

        # output_postp = output[:, :, :, :, :4].clone().detach()
        # print('xval:', output[0, 0, 0, 0, :4])
        # output_postp[:, :, :, :, 0] = (
        #     self.biases[:, :, :, :, 0] + output_postp[:, :, :, :, 0]) / (W*scale_factor)
        # output_postp[:, :, :, :, 1] = (
        #     self.biases[:, :, :, :, 1] + output_postp[:, :, :, :, 1]) / (H*scale_factor)
        # output_postp[:, :, :, :, 2] = (self.biases[:, :, :, :, 2] * torch.exp(
        #     output_postp[:, :, :, :, 2])) / W
        # output_postp[:, :, :, :, 3] = (self.biases[:, :, :, :, 3] * torch.exp(
        #     output_postp[:, :, :, :, 3])) / H
        # # write_tensor(output_postp.cpu(), 'preds.binlog', True)
        # # for e in output_postp.view(-1, 4):
        #     # print('x:{:.6f}, y:{:.6f}, w:{:.6f}, h:{:.6f}'.format(e[0], e[1], e[2], e[3]))

        # # # print(output_postp)
        # iou_target = target[:, :, 1:].clone().detach()
        # iou_target = iou_target.unsqueeze(1).expand(-1, H * W * N, -1,
        #                                             -1).contiguous()

        # ious = calculate_iou(
        #     output_postp.unsqueeze(4).expand(-1, -1, -1, -1, T,
        #                                      -1).contiguous().view(-1, 4),
        #     iou_target.view(-1, 4)).view(B, H, W, N, T)

        # iou_mask = ious.max(dim=4)[0] > 0.7
        # print(ious[iou_mask])
        # d_mask = torch.zeros(iou_mask.unsqueeze(4).shape).byte().to(
        #     output.device).expand(-1, -1, -1, -1, K)
        # d_mask[:, :, :, :, 4] = iou_mask

        # deltas[d_mask] = 0
        # print(d_mask)
        write_tensor(deltas.detach().cpu(), 'deltas.binlog', True)
        exit(0)
        return output.mean()

    def apply_sort_args(self, x, args, length):
        x2 = torch.zeros(x.shape)
        for index, arg in enumerate(args):
            x2[length * index:length * index +
               length] = x[arg * length:length * arg + length]
        return x2

    def calculate_biases(self, batch, height, width, n_anchors):
        biases = torch.zeros((batch, height, width, n_anchors, 4))
        for b in range(batch):
            for h in range(height):
                for w in range(width):
                    for n in range(n_anchors):
                        for i in range(4):
                            if i < 2:
                                biases[b, h, w, n, i] = w if i == 0 else h
                            else:
                                biases[b, h, w, n, i] = self.priors[2 * n +
                                                                    (i - 2)]
        return biases

class RegionLossV2(nn.Module):
    def __init__(self, priors, threshold=0.5, n_classes=20):
        super().__init__()

        self.priors = priors
        self.threshold = threshold
        self.n_classes = n_classes
        self.obj_scale = 5
        self.nobj_scale = 1

        self.biases = None
        self.seen = 0

    def forward(self, output, target):
        # print('output shape', output.shape)
        # print('target shape', target.shape)
        # print(target)

        K = 5 + self.n_classes
        B, Z, H, W = output.shape
        N = Z // K
        T = target.shape[1]

        output = output.permute((0, 2, 3, 1)).view(B, H, W, N, K).contiguous()
        deltas = torch.zeros(output.shape, device=output.device)
        # write_tensor(deltas.detach().cpu(), 'deltas.bin', True)

        deltas[:, :, :, :, 4] = self.nobj_scale * (0 - output[:, :, :, :, 4])
        # write_tensor(deltas.detach().cpu(), 'deltas.bin', True)
        avg_anyobj = output[:, :, :, :, 4].sum() / (H * W * N * B)

        if self.biases is None:
            self.biases = self.calculate_biases(B, H, W, N).to(output.device)

        output_postp = output[:, :, :, :, :4].clone().detach()
        output_postp[:, :, :, :, 0] = (
            self.biases[:, :, :, :, 0] + output_postp[:, :, :, :, 0]) / W
        output_postp[:, :, :, :, 1] = (
            self.biases[:, :, :, :, 1] + output_postp[:, :, :, :, 1]) / H
        output_postp[:, :, :, :, 2] = (self.biases[:, :, :, :, 2] * torch.exp(
            output_postp[:, :, :, :, 2])) / W
        output_postp[:, :, :, :, 3] = (self.biases[:, :, :, :, 3] * torch.exp(
            output_postp[:, :, :, :, 3])) / H

        iou_target = target[:, :, 1:].clone().detach()
        iou_target = iou_target.unsqueeze(1).expand(-1, H * W * N, -1,
                                                    -1).contiguous()

        ious = calculate_iou(
            output_postp.unsqueeze(4).expand(-1, -1, -1, -1, T,
                                             -1).contiguous().view(-1, 4),
            iou_target.view(-1, 4)).view(B, H, W, N, T)

        iou_mask = ious.max(dim=4)[0] > 0.6

        d_mask = torch.zeros(iou_mask.unsqueeze(4).shape).byte().to(
            output.device).expand(-1, -1, -1, -1, K)
        d_mask[:, :, :, :, 4] = iou_mask

        deltas[d_mask] = 0

        if self.seen < 12800:
            pass

        output = output.cpu()
        deltas = deltas.cpu()
        target = target.cpu()
        output_postp = output_postp.cpu()

        # write_tensor(deltas.detach(), 'deltas.bin', True)

        truth_boxes = target[:, :, 1:].clone().detach()
        truth_classes = target[:, :, 0].clone().detach()
        i = (truth_boxes[:, :, 0] * W).long()
        j = (truth_boxes[:, :, 1] * H).long()

        tbs_shifted = truth_boxes.clone()
        tbs_shifted[:, :, 0] = 0
        tbs_shifted[:, :, 1] = 0

        shift_mask = torch.zeros((B, T, H, W)).byte()

        for b in range(B):
            for t in range(T):
                shift_mask[b, t, j[b, t], i[b, t]] = 1

        output_shifted = output_postp.unsqueeze(1).expand(
            -1, T, -1, -1, -1, -1)[shift_mask].view(B, T, N, 4)
        output_shifted[:, :, :, :2] = 0

        bias_match = False
        if bias_match == True:
            pass

        ious = calculate_iou(
            tbs_shifted.unsqueeze(2).expand(-1, -1, 5, -1).contiguous().view(
                -1, 4), output_shifted.view(-1, 4)).view(B, T, 5)

        max_ious, max_indices = ious.max(dim=2)

        priors = torch.from_numpy(np.array(self.priors)).view(N, 2)
        priors_volume = torch.zeros((B, T, 2))

        for b in range(B):
            for t in range(T):
                if max_ious[b, t] != 0.0:
                    priors_volume[b, t] = priors[max_indices[b, t]]

        tx = truth_boxes[:, :, 0] * W
        ty = truth_boxes[:, :, 1] * H
        tx = tx - i.float()
        ty = ty - j.float()

        tw = torch.log(truth_boxes[:, :, 2] * W / priors_volume[:, :, 0])
        th = torch.log(truth_boxes[:, :, 3] * H / priors_volume[:, :, 1])
        tw[torch.isnan(tw)] = 0.0
        th[torch.isnan(th)] = 0.0

        scales = (1.0 * (2 - truth_boxes[:, :, 2] * truth_boxes[:, :, 3])
                  ).unsqueeze(2).expand(-1, -1, 4)
        t_mask = scales != 1.0
        p_mask = torch.zeros(deltas.shape).byte()
        o_mask = torch.zeros(deltas.shape).byte()
        c_mask = torch.zeros(deltas.shape).byte()

        for b in range(B):

            duplicate_dict = {}
            for t in range(T):
                if tx[b, t] == 0.0 and ty[b, t] == 0.0 and tw[
                        b, t] == 0.0 and th[b, t] == 0.0:
                    break
                cur_j = j[b, t].item()
                cur_i = i[b, t].item()
                max_n = max_indices[b, t].item()
                p_mask[b, cur_j, cur_i, max_n, :4] = 1
                o_mask[b, cur_j, cur_i, max_n, 4] = 1
                c_mask[b, cur_j, cur_i, max_n, 5:] = 1

                if tuple((cur_i, cur_j, max_n)) in duplicate_dict:
                    duplicate_dict[tuple((cur_i, cur_j, max_n))].append((b, t))
                else:
                    duplicate_dict[tuple((cur_i, cur_j, max_n))] = [(b, t)]

            for key in duplicate_dict.keys():
                items = duplicate_dict[key]
                if len(items) > 1:
                    for item in items[:-1]:
                        truth_boxes[item[0]][item[1]] = -1
                        target[item[0], item[1]] = -1
                        truth_classes[item[0]][item[1]] = -1
                        t_mask[item[0]][item[1]] = 0

        t_volume = torch.stack([tx, ty, tw, th], dim=2)
        t_volume = t_volume[t_mask]
        scales = scales[t_mask]

        # Sort t_volume & scales
        sorted_t_volume = torch.zeros(t_volume.shape)
        sorted_scales = torch.zeros(scales.shape)
        sorted_truth_classes = torch.zeros(
            truth_classes[truth_classes > -1].shape)
        sorted_t_i = 0

        for b in range(B):
            max_indices_batch = max_indices[b][target[b, :, 1] > -1]
            ts = truth_boxes[b][truth_boxes[b] > -1].view(-1, 4)
            i_s = (ts[:, 0] * W).long()
            j_s = (ts[:, 1] * H).long()
            ps = list(zip(j_s, i_s, max_indices_batch))
            sort_args = list(
                OrderedDict(sorted(enumerate(ps), key=lambda x: x[1])).keys())
            sub_sorted_t = self.apply_sort_args(
                t_volume[sorted_t_i * 4:4 * (sorted_t_i + len(sort_args))],
                sort_args, 4)
            sub_sorted_s = self.apply_sort_args(
                scales[sorted_t_i * 4:4 * (sorted_t_i + len(sort_args))],
                sort_args, 4)
            sub_sorted_c = self.apply_sort_args(
                truth_classes[b][truth_classes[b] > -1], sort_args, 1)
            sorted_t_volume[sorted_t_i * 4:4 *
                            (sorted_t_i + len(sort_args))] = sub_sorted_t
            sorted_scales[sorted_t_i * 4:4 *
                          (sorted_t_i + len(sort_args))] = sub_sorted_s
            sorted_truth_classes[sorted_t_i:(
                sorted_t_i + len(sort_args))] = sub_sorted_c
            sorted_t_i += len(sort_args)

        deltas[p_mask] = sorted_scales * (sorted_t_volume - output[p_mask])
        deltas[o_mask] = 5 * (1 - output[o_mask])

        rescore = False
        if rescore == True:
            pass

        background = False
        if background == True:
            deltas[o_mask] = 5 * (0 - output[o_mask])

        class_target = torch.zeros(output[c_mask].shape)
        class_scale = torch.tensor(1.0)

        ci = 0
        for cat in sorted_truth_classes:
            v = one_hot_encode(cat, self.n_classes)
            class_target[ci:ci + self.n_classes] = v
            ci += self.n_classes

        deltas[c_mask] = class_scale * (class_target - output[c_mask])
        cost = torch.sqrt((deltas**2).sum())**2

        # print('REGION LOSS:', cost)

        # print('Region Avg IOU: {}, Class: {},\n Obj: {}, No Obj: {},\n Avg Recall: {}, count: {}'.format(
        #     avg_iou/count, avg_cat/class_count, avg_obj/count, avg_anyobj, recall/count, count))

        return cost

    def apply_sort_args(self, x, args, length):
        x2 = torch.zeros(x.shape)
        for index, arg in enumerate(args):
            x2[length * index:length * index +
               length] = x[arg * length:length * arg + length]
        return x2

    def calculate_biases(self, batch, height, width, n_anchors):
        biases = torch.zeros((batch, height, width, n_anchors, 4))
        for b in range(batch):
            for h in range(height):
                for w in range(width):
                    for n in range(n_anchors):
                        for i in range(4):
                            if i < 2:
                                biases[b, h, w, n, i] = w if i == 0 else h
                            else:
                                biases[b, h, w, n, i] = self.priors[2 * n +
                                                                    (i - 2)]
        return biases


import argparse
import requests

from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)

def show_banner():
	print('''
 ____  _      ____                      _
|  _ \\(_)_ __/ ___|  ___  __ _ _ __ ___| |__
| | | | | '__\___ \ / _ \/ _` | '__/ __| '_ \\
| |_| | | |   ___) |  __/ (_| | | | (__| | | |
|____/|_|_|  |____/ \___|\\__,_|_|  \\___|_| |_|

	--[ Coded by : King D a.k.a N3utr0n
	--[ Team : FHC (FR13NDs Hackers Club)
	''')

def readFile(filename):
	try:
		filename = open(filename, 'r').read().split('\n')
		return filename
	except:
		print('[-] Erro ao abrir o arquivo.')
		return False

def make_url(url, uri):
	if url.endswith('/'):
		return url + uri
	else:
		return url + '/' + uri

def return_code(url):
	try:
		return requests.get(url, verify=False, allow_redirects=False).status_code
	except:
		pass

def DirSearch(url, filename, sn):
	print('[+] Target URL :', url)
	print('[!] File :', filename)
	print('[*] Searching directories and files...\n')

	filename = readFile(filename)
	if filename:
		for uri in filename:
			target_url = make_url(url, uri)
			code = return_code(target_url)
			if code == 200:
				print('\033[1;32m[+]\033[m Code : 200 OK | Found :', target_url,'\033[m')
			elif code == 302 or code == 303:
				print('\033[1;35m[!]\033[m Code :', code, ' | Moved :', target_url,'\033[m')
			if sn:
				print('\033[1;31m[-]\033[m Code : 404 | Not Found :\033[31m', target_url,'\033[m')
		print('\n[!] Done...')
	else:
		return False

show_banner()
parser = argparse.ArgumentParser(description='This tool search for directories and files.')
parser.add_argument('-u', '--url', help='Target URL.', required=True)
parser.add_argument('-f', '--filename', help='Filename that contains the URI.')
parser.add_argument('-sn', '--show-notfound', help='Show result not found', action='store_true')
parser.add_argument('-r', '--run', help='Running tool.', action='store_true')
args = parser.parse_args()

if args.run:
	if args.filename == None:
		args.filename = 'common.txt'

	DirSearch(args.url, args.filename, args.show_notfound)


import json
from datetime import datetime
from typing import Union, Dict, Any

from nonebot.adapters.onebot.v11 import Bot, GroupMessageEvent, PrivateMessageEvent

group_id = 111111
user_id = 222222

class NewNumber:
    _self_id = 330000
    _message_id = 1

    @classmethod
    def self_id(cls) -> int:
        cls._self_id += 1
        return cls._self_id

    @classmethod
    def message_id(cls) -> int:
        cls._message_id += 1
        return cls._message_id

# https://github.com/howmanybots/onebot/blob/master/v11/specs/event/message.md#%E7%A7%81%E8%81%8A%E6%B6%88%E6%81%AF
def generate_private_message(self_id: Union[int, str], message: str) -> Dict[str, Any]:
    return {
        "time": int(datetime.now().timestamp()),
        "self_id": int(self_id),
        "post_type": "message",
        "message_type": "private",
        "sub_type": "friend",
        "message_id": NewNumber.message_id(),
        "user_id": user_id,
        "message": message,
        "raw_message": message,
        "font": 0,
        "sender": {
            "user_id": user_id,
            "nickname": "nickname"
        }
    }

# https://github.com/howmanybots/onebot/blob/master/v11/specs/event/message.md#%E7%BE%A4%E6%B6%88%E6%81%AF
def generate_group_message(self_id: Union[int, str], message: str) -> Dict[str, Any]:
    return {
        "time": int(datetime.now().timestamp()),
        "self_id": int(self_id),
        "post_type": "message",
        "message_type": "group",
        "sub_type": "normal",
        "message_id": NewNumber.message_id(),
        "group_id": group_id,
        "user_id": user_id,
        "anonymous": None,
        "message": message,
        "raw_message": message,
        "font": 0,
        "sender": {
            "user_id": user_id,
            "nickname": "nickname"
        }
    }

async def handle_private_message(bot: Bot, message: str) -> None:
    event = PrivateMessageEvent.parse_obj(generate_private_message(bot.self_id, message))
    return await bot.handle_event(event)

async def handle_group_message(bot: Bot, message: str) -> None:
    event = GroupMessageEvent.parse_obj(generate_group_message(bot.self_id, message))
    return await bot.handle_event(event)


from typing import Iterator

import requests
from base import LotSpaces, Scraper

class NJITScraper(Scraper):
    """Scrape NJIT json.

    https://mobile.njit.edu/parking/
    """

    HOME_URL = "https://mobile.njit.edu/parking/"
    API_URL = "http://mobile.njit.edu/parking/data.php"
    TIMEOUT = 5

    name = "njit"

    def fetch_spaces(self) -> Iterator[LotSpaces]:
        session = requests.Session()
        session.get(self.HOME_URL, headers={"User-Agent": "open-parking-spaces"})
        response = session.post(
            self.API_URL, timeout=self.TIMEOUT, headers={"Referer": self.HOME_URL}
        )
        response.raise_for_status()
        data = response.json()
        for row in data["decks"].values():
            yield LotSpaces(
                lot=row["SiteName"],
                spaces=int(row["Available"]),
                capacity=int(row["Total"]),
                address=row["Address"],
            )


from itertools import permutations
def get():
    a=list(map(int,input().split()))
    return list(permutations(a))

n=int(input())

s=get()
# on even - 0 0|| 0 1 || 1 1
for _ in range(n-1):
    tr=get()
    m=float('inf')
    qq=[]
    for i in s:
        for j in tr:
            qq.append([i[0]+j[0],i[1]+j[1],i[2]+j[2]])
    qq.sort(key=lambda x:x[2])
    ss=[[m,m,m]]*3
    for i in qq:
        k=i[0]%2+i[1]%2
        ss[k]=min(ss[k],i,key=lambda x:x[2])
    s=[x for x in ss if x!=[m,m,m]]
print(s[1])


from  PIL import Image, ImageDraw
import numpy as np
import face_recognition

# Load first sample picture and learn how to recognize it
ronaldo_image = face_recognition.load_image_file("imgs/known/Ronaldo.jpg");
ronaldo_face_encodings = face_recognition.face_encodings(ronaldo_image)[0];

# Load second sample picture and learn how to recognize it
dybala_image = face_recognition.load_image_file("imgs/known/Paulo Dybala.jpg");
dybala_face_encodings = face_recognition.face_encodings(dybala_image)[0];

known_face_encodings = [
  ronaldo_face_encodings,
  dybala_face_encodings
]

known_face_names = [
  "Cristiano Ronaldo",
  "Dybala",
]

# Load an image with an unknown face
unknown_image = face_recognition.load_image_file("imgs/groups/juventusfc.jpg");
#Find all the faces and face encodings in unknown image
face_locations = face_recognition.face_locations(unknown_image);
face_encodings = face_recognition.face_encodings(unknown_image,face_locations);
pil_image = Image.fromarray(unknown_image);
draw = ImageDraw.Draw(pil_image);

for (top, right, bottom, left), face_encoding in zip(face_locations, face_encodings) :
  # See if the face is a match for the known face(s)
  matches = face_recognition.compare_faces(known_face_encodings, face_encoding);
  name = "Unknown";
  face_distance = face_recognition.face_distance(known_face_encodings, face_encoding);
  best_match_index = np.argmin(face_distance);
  if matches[best_match_index] :
    name = known_face_names[best_match_index];

  # Draw a box around the face using the Pillow module
  draw.rectangle(((left, top), (right, bottom)), outline=(0,0,255));

  #  Draw a label with a name below the face
  text_width, text_height = draw.textsize(name);
  draw.rectangle(((left, bottom -text_height - 10), (right, bottom)), fill= (0,0,255), outline=(0,0,255));
  draw.text((left+6, bottom- text_height- 5), name, fill=(255,255,255,255));
#delete draw
del draw;

#show image
pil_image.show();

pil_image.save("identify_and_draw_boxes_faces.jpg")


from receiver_server import *
from os import path
import sys
import pygtlink
import os
import imfusion
import numpy as np
import time
from scipy.signal import butter, filtfilt
import pandas as pd

SERVER_ADDRESS = "127.0.0.1"
PORT = 9003

v_data_dir = "/media/maria/Elements/Maria/DataBases/SpineIFL/SpinousProcessDb/SecondAcquisitionSession"
v_subject_name = "MariaT_F10"

USE_FORCE = False

def find_valid_indexes(y):
    smoothed_vel = np.squeeze(smooth(np.diff(y), 10))

    max_vel_pos = np.argmax(smoothed_vel)
    max_vel = smoothed_vel[max_vel_pos]
    first_half = np.flip(smoothed_vel[0:max_vel_pos])

    try:
        p1 = max_vel_pos - np.where(first_half<(0.1*max_vel))[0][0]

        second_half = smoothed_vel[max_vel_pos::]
        p2 = max_vel_pos + np.where(second_half < (0.1 * max_vel))[0][0]
    except:
        return 0, len(smoothed_vel) - 1, smoothed_vel

    return p1, p2, smoothed_vel

def smooth(signal, N=30):
    N = 2  # Filter order
    Wn = 0.05  # Cutoff frequency
    B, A = butter(N, Wn, output='ba')
    # Second, apply the filter
    filt = filtfilt(B, A, signal)
    return filt

def send_handshake(streamer_client, command_client):
    status_msg_1 = pygtlink.StatusMessage()
    status_msg_1.setDeviceName("StreamerSocket")
    status_msg_1.pack()
    streamer_client.connectToServer(SERVER_ADDRESS, PORT)
    streamer_client.send(status_msg_1.header + status_msg_1.body)

    status_msg_2 = pygtlink.StatusMessage()
    status_msg_2.setDeviceName("CommandSocket")
    status_msg_2.pack()
    command_client.connectToServer(SERVER_ADDRESS, PORT)
    command_client.send(status_msg_2.header + status_msg_2.body)

def get_data(sweep_path, traj_path):

    if not os.path.exists(traj_path):
        raise RuntimeError("Trajectory path not found: not existing dir: {}".format(traj_path))

    if not os.path.exists(sweep_path):
        raise RuntimeError("Trajectory path not found: not existing dir: {}".format(sweep_path))

    logging.info("\n**************\nSweep File: {} \nTrajectory File: {}".format(sweep_path, traj_path))
    imageset = imfusion.open(sweep_path)

    sweep = np.squeeze(np.array(imageset[0]))

    csv_dframe = pd.read_csv(traj_path)

    # if there is only one column probably the delimiter is ;
    if len(csv_dframe.columns) < 2:
        csv_dframe = pd.read_csv(traj_path, delimiter=";")

    y_robot = np.array(csv_dframe["y"])
    p_i, p_e, vel = find_valid_indexes(y_robot)

    # if force data are present, load them as well
    if "y_force" in csv_dframe.columns:
        y_force = np.array(csv_dframe["y_force"])
        logging.info("Force data present and correctly loaded")
    else:
        y_force = np.zeros(y_robot.shape)
        logging.info("Force data not present - zeros vector loaded")

    sweep = sweep[p_i:p_e, :, :]
    y_robot = y_robot[p_i:p_e]
    y_force = y_force[p_i:p_e]
    return sweep, y_robot, y_force

def send_data(streamer_client, sweep, y_pos, force_data = None, use_force = False):
    for (frame, y, force) in zip(sweep, y_pos, force_data):

        frame = np.squeeze(frame)

        matrix = np.eye(4)
        matrix[1, 3] = y

        img_msg = pygtlink.ImageMessage2()
        img_msg.setData(frame)
        img_msg.setSpacing([1, 1, 1])
        img_msg.setMatrix(matrix)

        sensor_msg = pygtlink.SensorMessage()
        sensor_msg.setData([0, force, 0])
        sensor_msg.setLength(3)

        img_msg.pack()
        streamer_client.send(img_msg.header + img_msg.body)

        sensor_msg.pack()
        streamer_client.send(sensor_msg.header + sensor_msg.body)
        time.sleep(0.03)

    # todo: send step done message

def main():

    streamer_client = pygtlink.ClientSocket()
    command_client = pygtlink.ClientSocket()
    imfusion.init()

    while True:
        a = input("Chose step: SACRUM DETECTION [s], VERTEBRAL COUNTING [v], FACET DETECTION [f]")

        if "s" in a:
            print("Streaming Sacrum Data")

        elif "v" in a:
            print("Streaming Spinous Process Data")
            sweep_path = os.path.join(v_data_dir, v_subject_name + ".imf")
            traj_path = os.path.join(v_data_dir, v_subject_name + ".csv")
            sweep, y_pos, force_data = get_data(sweep_path, traj_path)

            status_msg = pygtlink.StatusMessage()
            status_msg.setMessa

            sensor_msg.setData([0, force, 0])
            sensor_msg.setLength(3)

        elif "f" in a:
            print("Streaming Facet Detection Data")

if __name__ == "__main__":
    main()


from typing import List
from flask import Flask
import numpy as np
from flask import jsonify,request
import joblib

import json
app=Flask(__name__)

#helper
def trim(encodings_classes):
    classes=[]
    for i in encodings_classes:
        classes.append(i.strip())
    return classes

@app.route('/')
def home():
    return "Hello"

# Actual
@app.route('/api/v1/preprocess',methods=['GET','POST'])

def predict():
    # print(encoderSymptom_1.i(request.form['Symptom1']))
    # print(encoderSymptom_2)

    symptom1=encoderSymptom_1.index(request.form['Symptom1'])
    print(request.form['Symptom1'])
    symptom2=encoderSymptom_2.index(request.form['Symptom2'])
    symptom3=encoderSymptom_3.index(request.form['Symptom3'])
    symptom4=encoderSymptom_4.index(request.form['Symptom4'])

    symptom_array=np.array([symptom1,symptom2,symptom3,symptom4]).reshape(1,-1)
    print(symptom_array)
    prediction = model.predict(symptom_array)
    probability=np.max(model.predict_proba(symptom_array))
    prrediction=disease_encoder.inverse_transform(prediction)[0]

    return jsonify({
        'prediction':prrediction,
        'probability':probability
    })

if __name__=='__main__':
    encoderSymptom_1=joblib.load('./encoderSymptom_1').classes_
    encoderSymptom_2=joblib.load('./encoderSymptom_2').classes_
    encoderSymptom_3=joblib.load('./encoderSymptom_3').classes_
    encoderSymptom_4=joblib.load('./encoderSymptom_4').classes_
    encoderSymptom_1=trim(encoderSymptom_1)
    encoderSymptom_2=trim(encoderSymptom_2)
    encoderSymptom_3=trim(encoderSymptom_3)
    encoderSymptom_4=trim(encoderSymptom_4)

    disease_encoder=joblib.load('./encoder_disease')
    model=joblib.load('./model_v2')

    app.run(debug=True,port=80)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import requests
import socket
try:
    from urllib.parse import urljoin
except ImportError:
    from urlparse import urljoin
import dateutil.parser

class PyPSSL(object):

    def __init__(self, base_url='https://www.circl.lu/', api_version=2, basic_auth=None, auth_token=None):
        self.base_url = base_url
        self.api_version = api_version

        self.session = requests.Session()
        if basic_auth is not None:
            # basic_auth has do be a tuple ('user_name', 'password')
            self.session.auth = basic_auth
        elif auth_token is not None:
            self.session.headers.update({'Authorization': auth_token})
        else:
            # No authentication defined.
            pass

    def _query(self, url, timeout=None):
        response = self.session.get(url, timeout=timeout)
        try:
            return response.json()
        except:
            raise Exception('Unable to decode JSON object: ' + response.text)

    def _check_IP(self, ip):
        if ':' in ip:
            return {'error': 'IPv6 is not (yet) supported'}
        splitted = ip.split('/')
        try:
            if len(splitted) == 2:
                ip, block = splitted
                if int(block) < 23:
                    return {'error': 'Maximum CIDR block size reached >/23'}
            socket.inet_aton(ip)
        except:
            return {'error': 'Incorrect format'}
        return None

    def query(self, q, timeout=None):
        check = self._check_IP(q)
        if check is not None:
            return check
        if self.api_version == 1:
            path = 'pssl/query/{}'.format(q)
        else:
            path = 'v2pssl/query/{}'.format(q)
        return self._query(urljoin(self.base_url, path), timeout=timeout)

    def query_cert(self, q, timeout=None):
        if self.api_version != 2:
            return {'error': 'Only available in API v2'}
        path = 'v2pssl/cquery/{}'.format(q)
        return self._query(urljoin(self.base_url, path), timeout=timeout)

    def fetch_cert(self, q, make_datetime=True, timeout=None):
        if self.api_version != 2:
            return {'error': 'Only available in API v2'}
        path = 'v2pssl/cfetch/{}'.format(q)
        response = self._query(urljoin(self.base_url, path), timeout=timeout)
        if response.get('error') or not make_datetime:
            return response
        # create python datetime, doesn't return a json object
        response['info']['not_before'] = dateutil.parser.parse(response['info']['not_before'])
        response['info']['not_after'] = dateutil.parser.parse(response['info']['not_after'])
        return response


import abc

from kani.ai_function import AIFunction
from kani.exceptions import MissingModelDependencies
from kani.models import ChatMessage
from ..base import BaseEngine, Completion

try:
    from ctransformers import AutoModelForCausalLM
except ImportError as e:
    raise MissingModelDependencies(
        'The CTransformersEngine requires extra dependencies. Please install kani with "pip install'
        ' kani[ctransformers]".'
    ) from None

class CTransformersEngine(BaseEngine, abc.ABC):
    """
    This class implements the main decoding logic for any GGML model based on a pretrained ``AutoModelForCausalLM``.

    **GPU Support**

    In order to load a model on GPU, the underlying GGML model must support CUDA. To see a list of supported models,
    see `this table <https://github.com/marella/ctransformers/tree/main#supported-models>`_.

    .. caution::
        If your model supports CUDA, you must also install additional CUDA dependencies. Run
        ``pip install 'ctransformers[cuda]'`` if you have not installed CUDA dependencies elsewhere (e.g. through
        torch).

    To load some or all of the model layers on GPU, pass ``gpu_layers=...`` in the ``model_load_kwargs``.
    """

    def __init__(
        self,
        model_id: str,
        model_file: str = None,
        max_context_size: int = 1024,
        model_load_kwargs: dict = None,
        **hyperparams,
    ):
        """
        :param model_id: The ID of the model to load from HuggingFace or locally.
        :param model_file: The file of the model to load from HuggingFace repo or locally.
        :param max_context_size: The context size of the model.
        :param model_load_kwargs: Additional arguments to pass to ``AutoModelForCausalLM.from_pretrained()``.
            See `this link <https://github.com/marella/ctransformers/tree/main#documentation>`_ for more info.
        :param hyperparams: Additional arguments to supply the model during generation.
        """

        if model_load_kwargs is None:
            model_load_kwargs = {}

        self.model_id = model_id
        self.model_file = model_file
        self.max_context_size = max_context_size
        model_load_kwargs.setdefault("context_length", self.max_context_size)
        self.model = AutoModelForCausalLM.from_pretrained(model_id, model_file=model_file, **model_load_kwargs)
        self.hyperparams = hyperparams

    @abc.abstractmethod
    def build_prompt(self, messages: list[ChatMessage], functions: list[AIFunction] | None = None) -> str | list[int]:
        """Given the list of messages from kani, build either a single string representing the prompt for the model,
        or build the token tensor."""
        raise NotImplementedError

    async def predict(
        self, messages: list[ChatMessage], functions: list[AIFunction] | None = None, **hyperparams
    ) -> Completion:
        """
        Given the current context of messages and available functions, get the next predicted chat message from the LM.

        :param messages: The messages in the current chat context. ``sum(message_len(m) for m in messages)`` is
            guaranteed to be less than max_context_size.
        :param functions: The functions the LM is allowed to call.
        :param hyperparams: Any additional parameters to pass to GenerationMixin.generate(). (See
            https://github.com/marella/ctransformers#method-llmgenerate)
        """
        prompt = self.build_prompt(messages, functions)
        if isinstance(prompt, str):
            # prompt str to tokens
            input_toks = self.model.tokenize(prompt)
            input_len = len(input_toks)
        elif isinstance(prompt, list):
            input_toks = prompt
            input_len = len(input_toks)
        else:
            raise TypeError("build_prompt should either return a str or a list[int].")

        output_toks = []
        output_len = 0
        for tok in self.model.generate(input_toks, **hyperparams):
            output_toks.append(tok)
            output_len += 1
            # ctransformers does not automatically stop at end of context length
            # (e.g. https://github.com/zhudotexe/kani/actions/runs/6152842183/job/16695721588)
            # so we force it to stop if we would bust the context length
            if input_len + output_len >= self.max_context_size:
                break
        content = self.model.detokenize(output_toks).strip()

        return Completion(ChatMessage.assistant(content), prompt_tokens=input_len, completion_tokens=output_len)


"""
Engine Module

It is an engine, it runs. Puts piped data in buffer, spawns processes for the aggregation, transformation and storage of data.

Contains...
- class EngineThread:
manages buffer, moves data from buffer to processing queue
- class Processor:
loads json from data messages to python, runs data transformation + storage process
"""

from multiprocessing import Process, JoinableQueue
from queue import Queue
from threading import Thread
from time import time
from datetime import datetime
import numpy as np
from .processor import Processor
from .data_puller import DataPuller
from strom.dstream.dstream import DStream
from strom.utils.logger.logger import logger

__version__ = "0.1"
__author__ = "Molly <molly@tura.io>"

class Engine(Process):
    """
    Based off `threading.Thread`
    Instantiated in server

    Contains buffer, queue for processors, processors, ConsumerThread.
    """

    def __init__(self, engine_conn, processors=4, buffer_roll=0, buffer_max_batch=50, buffer_max_seconds=1, test_mode=False, test_outfile='engine_test_output/engine_test_output'):
        """
        Initializes with empty buffer & queue,
         set # of processors...
        :param processors: number of processors to start
        :type processors: int
        """
        logger.info("Initializing EngineThread")
        super().__init__()
        self.test_run = test_mode
        self.test_outfile = test_outfile
        self.test_batches = {}
        self.pipe_conn = engine_conn
        self.buffers_out_q = JoinableQueue()
        self.number_of_processors = processors
        self.processors = []
        self.run_engine = False
        self.buffer_record_limit = int(buffer_max_batch)
        self.buffer_time_limit_s = float(buffer_max_seconds)
        self.buffers = {}
        self.buffer_in_qs = {}
        self.buffer_workers = {}
        self.data_pullers = {}
        self.buffer_roll = -buffer_roll
        if buffer_roll > 0:
            self.buffer_roll_index = -buffer_roll
        else:
            self.buffer_roll_index = None

    def _init_processors(self):
        """Initializes + starts set number of processors"""
        for n in range(self.number_of_processors):
            processor = Processor(self.buffers_out_q, self.test_run)
            processor.start()
            self.processors.append(processor)

    def _new_buffer(self, partition_key):
        if partition_key not in self.buffers:
            self.buffers[partition_key] = np.array(
                [{0: 0}] * (self.buffer_record_limit * self.number_of_processors)).reshape(
                self.number_of_processors, self.buffer_record_limit)
            if self.test_run:
                self.test_batches[partition_key] = 1
            if partition_key not in self.buffer_in_qs:
                self.buffer_in_qs[partition_key] = Queue()
            else:
                logger.warn(f"New buffer, existing buffer_in_q for stream {partition_key}")
            if partition_key not in self.buffer_workers:
                self.buffer_workers[partition_key] = Thread(target=self.run_buffer,
                                                            args=[partition_key])
                self.buffer_workers[partition_key].start()
            else:
                logger.warn(f"New buffer, existing buffer_worker for stream {partition_key}")
            return True
        else:
            return False

    def _new_data_puller(self, partition_key, template):
        if partition_key not in self.data_pullers:
            if partition_key in self.buffer_in_qs:
                self.data_pullers[partition_key] = DataPuller(template, self.buffer_in_qs[partition_key])
                self.data_pullers[partition_key].start()
            else:
                raise ValueError(f"Attempting to init data_puller for stream {partition_key}, buffer input queue for stream does not exist")
            return True
        else:
            return False

    def run(self):
        """
        Sets up numpy array buffer and puts stuff in and gets stuff out
        """
        self._init_processors()
        self.run_engine = True

        while self.run_engine:
            if self.pipe_conn.poll():
                item = self.pipe_conn.recv()
                # branch 2 - stop engine
                if item == "stop_poison_pill":
                    for q in self.buffer_in_qs.keys():
                        self.buffer_in_qs[q].put("stop_buffer_worker")
                    self.run_engine = False
                    break
                # branch 1 - engine running, good data
                elif type(item) is tuple:
                    partition_key = item[0]['stream_token']
                    new_buffer = self._new_buffer(partition_key)
                    if new_buffer:
                        logger.info(f"Initialized buffer for stream {partition_key}")
                    if item[1] == "new":
                        if item[0]["data_rules"]["pull"] is True:
                            new_puller = self._new_data_puller(partition_key, item[0])
                            if new_puller:
                                print(f"Initialized data puller for stream {partition_key}")
                            else:
                                logger.warn(
                                    f"Attempting to initialize data puller for stream {partition_key} - puller already exists")
                    elif item[1] == "load":
                        self.buffer_in_qs[partition_key].put(item[0])
                    else:
                        raise TypeError(
                            "Invalid tuple in pipe - index 1 must be str 'load' or str 'new'")
                else:
                    raise TypeError("Invalid item in pipe")
        logger.info("Terminating Engine Thread")
        self.stop_engine()

    def run_buffer(self, partition_key):
        last_col = self.buffer_record_limit - 1
        last_row = self.number_of_processors - 1
        cur_row = 0
        cur_col = 0
        batch_tracker = {'start_time': time(), 'leftos_collected': False}

        while self.run_engine:
            try:
                item = self.buffer_in_qs[partition_key].get(timeout=self.buffer_time_limit_s)
                # branch 2 - stop engine
                if item == "stop_buffer_worker":
                    break
                # branch 1 - engine running, good data
                elif isinstance(item, DStream) or (type(item) is dict and "stream_token" in item.keys()):
                    if "data_rules" in item.keys(): # some unit test data doesnt have this field
                        if "date_format" in item["data_rules"].keys():
                            if item["data_rules"]["date_format"] is not None:
                                item["timestamp"] = datetime.strptime(item["timestamp"], item["data_rules"]["date_format"]).timestamp()
                    # branch 1.1 - not last row
                    if cur_row < last_row:
                        # branch 1.1a - not last column, continue row
                        if cur_col < last_col:
                            logger.info("Buffering- row {}".format(cur_row))
                            self.buffers[partition_key][cur_row, cur_col] = item
                            cur_col += 1
                        # branch 1.1b - last column, start new row
                        else:
                            self.buffers[partition_key][cur_row, cur_col] = item
                            if self.test_run:
                                self.buffers_out_q.put((self.buffers[partition_key][cur_row].copy(), f"{self.test_outfile}_{partition_key}_{self.test_batches[partition_key]}.txt"))
                                self.test_batches[partition_key] += 1
                            else:
                                self.buffers_out_q.put(self.buffers[partition_key][cur_row].copy())
                            logger.info("New batch queued")
                            roll_window = self.buffers[partition_key][cur_row, self.buffer_roll_index:]
                            cur_row += 1
                            for n in roll_window:
                                for i in range(abs(self.buffer_roll)):
                                    self.buffers[partition_key][cur_row, i] = n
                            cur_col -= cur_col + self.buffer_roll
                            # REMOVE
                            batch_tracker['start_time'] = time()
                    # branch 1.2 - last row
                    else:
                        # branch 1.2a - not last column, continue row
                        if cur_col < last_col:
                            self.buffers[partition_key][cur_row, cur_col] = item
                            cur_col += 1
                        # branch 1.2b - last column, start return to first row in new cycle
                        else:
                            self.buffers[partition_key][cur_row, cur_col] = item
                            if self.test_run:
                                self.buffers_out_q.put((self.buffers[partition_key][cur_row].copy(), f"{self.test_outfile}_{partition_key}_{self.test_batches[partition_key]}.txt"))
                                self.test_batches[partition_key] += 1
                            else:
                                self.buffers_out_q.put(self.buffers[partition_key][cur_row].copy())

                            roll_window = self.buffers[partition_key][cur_row, self.buffer_roll_index:]
                            cur_row -= cur_row
                            for n in roll_window:
                                for i in range(abs(self.buffer_roll)):
                                    self.buffers[partition_key][cur_row, i] = n
                            cur_col -= cur_col + self.buffer_roll
                            batch_tracker['start_time'] = time()
                    batch_tracker['leftos_collected'] = False
                # branch 3 bad data
                else:
                    raise TypeError("Queued item is not valid dictionary.")
            except:
            # buffer time max reached, engine still running
                logger.info("Buffer batch timeout exceeded")
                if self.run_engine is True:
                    # engine running, batch timeout with new buffer data (partial row)
                    if cur_col > abs(self.buffer_roll) and batch_tracker['leftos_collected'] is False:
                        logger.info(
                            "Collecting leftovers- pushing partial batch to queue after batch timeout")
                        if self.test_run:
                            self.buffers_out_q.put((self.buffers[partition_key][cur_row, :cur_col].copy(), f"{self.test_outfile}_{partition_key}_{self.test_batches[partition_key]}.txt"))
                            self.test_batches[partition_key] += 1
                        else:
                            self.buffers_out_q.put(self.buffers[partition_key][cur_row, :cur_col].copy())
                        if cur_row < last_row:
                            cur_row += 1
                        else:
                            cur_row -= cur_row

                        cur_col -= cur_col
                        batch_tracker['start_time'] = time()
                        batch_tracker['leftos_collected'] = True
                    # leftovers already collected
                    else:
                        logger.info("No new data- resetting batch timer")
                        batch_tracker['start_time'] = time()

    def stop_engine(self):
        self.pipe_conn.close()
        if self.run_engine is True:
            self.run_engine = False
        for p in self.data_pullers.keys():
            self.data_pullers[p].pulling = False
        logger.info(self.buffers_out_q.qsize())
        self.buffers_out_q.join()
        logger.info("Queue joined")
        for p in self.processors:
            logger.info("Putting poison pills in Q")
            self.buffers_out_q.put("666_kIlL_thE_pROCess_666")
        logger.info("Poison pills done")
        for p in self.processors:
            p.join()
            logger.info("Engine shutdown- processor joined")
        print("done")


"""added google calendar ID field to the room event model

Revision ID: da573c763b9c
Revises: 32d2449aee31
Create Date: 2019-02-11 17:23:16.278409

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = 'da573c763b9c'
down_revision = '32d2449aee31'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('scheduler_room_reservations', sa.Column('google_calendar_id', sa.String(length=255), nullable=True))
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_column('scheduler_room_reservations', 'google_calendar_id')
    # ### end Alembic commands ###


#!/usr/bin/python
#Encoding UTF-8
import fileinput
import json

def count_words(json_input):
    doc = json.loads(json_input)

    text = doc["text"]
    parts = text.split(" ")
    doc["count"] = len(parts)
    return json.dumps(doc)

if __name__ == '__main__':
    for line in fileinput.input():
        print("%s") % (count_words(line))


import geopandas as gpd
import numpy as np
import pandas as pd
import earthpy.plot as ep
from matplotlib import pyplot as plt
from matplotlib.colors import ListedColormap
import logging
import tqdm
"""
    This class is used for various NVDI calculations.

    For more information check:
    https://en.wikipedia.org/wiki/Enhanced_vegetation_index

"""
def aggregate_ndvi_habitat(ndvi_geo_df: gpd.geodataframe.GeoDataFrame) -> pd.Series:
    """
    Calculate the aggregated statistics for NDVI
    @param ndvi_geo_df: geopandas dataframe, with ndvi values for each pixel
    @return pandas series: with 'mean', 'std', 'min', 'max', 'count' in the row axis
    """
    return ndvi_geo_df['ndvi'].agg(['mean', 'std', 'min', 'max', 'count'])

def generate_ndvi_channel(tile):
        """
        Generate ndvi channel from 2 bands.

        @param tile: rgbi tile to calculate to the NDVI from.
        @return a NDVI channel.
        """
        print("Generating NDVI channel...")
        red = tile[0]
        nir = tile[3]
        ndvi = []

        # No more numpy way for looping through these array's which lead to not good ndvi calculation.
        # Now we loop through each pixel directly
        for i in tqdm.tqdm(range(len(red))):
            ndvi_x = []
            for x in range(len(red[i])):
                upper_ndvi = (int(nir[i][x])-int(red[i][x]))
                lower_ndvi = (int(nir[i][x])+int(red[i][x]))

                if lower_ndvi == 0:
                    ndvi_x.append(0)
                else:
                    ndvi_cur = upper_ndvi/lower_ndvi
                    ndvi_cur = (ndvi_cur*100)+100
                    ndvi_x.append(int(ndvi_cur))

            ndvi.append(ndvi_x)
        return np.array(ndvi)

def normalized_diff(b1: np.array, b2: np.array) -> np.array:
    """Take two n-dimensional numpy arrays and calculate the normalized
    difference.

    Math will be calculated (b1-b2) / (b1 + b2). b1 is NIR, b2 is Red

    Parameters
    ----------
    b1, b2 : numpy arrays
        Two numpy arrays that will be used to calculate the normalized
        difference. Math will be calculated (b1-b2) / (b1+b2).

    Returns
    ----------
    n_diff : numpy array
        The element-wise result of (b1-b2) / (b1+b2) calculation. Inf values
        are set to nan. Array returned as masked if result includes nan values.
    """
    logging.info('Calculating NDVI')
    if not (b1.shape == b2.shape):
        raise ValueError("Both arrays should have the same dimensions")

    b1 = b1.astype('f4')
    b2 = b2.astype('f4')
    # Ignore warning for division by zero
    with np.errstate(divide="ignore"):#np.seterr(divide='ignore', invalid='ignore')
        n_diff = (b1 - b2) / (b1 + b2)

    # Set inf values to nan and provide custom warning
    if np.isinf(n_diff).any():
        warnings.warn(
            "Divide by zero produced infinity values that will be replaced "
            "with nan values",
            Warning,
        )
        n_diff[np.isinf(n_diff)] = np.nan

    # Mask invalid values
    if np.isnan(n_diff).any():
        n_diff = np.ma.masked_invalid(n_diff)

    return n_diff

def enhanced_vegetation_index(red: np.array, blue: np.array, nir: np.array, L: float=1, c1: float=6, c2: float=7.5, G: float=2.5) -> np.array:
    """
    DEPRECATED MIGHT BE USED IN THE FUTURE!

    This function makes groups out of the NVDI values.
    For a nicer plot.

    @param red: numpy array color values for red channel
    @param blue: numpy array color values for blue channel
    @param nir: numpy array values for infra-red channel
    @param L: float see Wikipedia
    @param c1: float see Wikipedia
    @param c2: float see Wikipedia
    @param G: float see Wikipedia
    @return numpy array, vegetation index for each pixel
    """
    if not ((red.shape == blue.shape) and (blue.shape == nir.shape)):
        raise ValueError("All arrays should have the same dimensions")
    # Ignore warning for division by zero
    red = red.astype('f4')
    blue = blue.astype('f4')
    nir = nir.astype('f4')
    with np.errstate(divide="ignore"):#np.seterr(divide='ignore', invalid='ignore')
        evi = G * (nir - red) / (nir + c1 * red - c2 * blue + L)

    # Set inf values to nan and provide custom warning
    if np.isinf(evi).any():
        warnings.warn(
            "Divide by zero produced infinity values that will be replaced "
            "with nan values",
            Warning,
        )
        evi[np.isinf(evi)] = np.nan

    # Mask invalid values
    if np.isnan(evi).any():
        evi = np.ma.masked_invalid(evi)

    return evi

def make_ndvi_plot(path, title, plot, save_figure =True):
    """ Plot NVDI data

    @path: path to the numpy array with the NVDI data.
    @title: A title on the plot.
    @plot: bool for whether to see the plots in de output or not
    """
    data_ndvi = np.load(path, allow_pickle=True)
    #plot ndvi
    if plot:
        ep.plot_bands(data_ndvi,
                figsize=(28, 12),
                cmap='RdYlGn',
                scale=False,
                vmin=-1, vmax=1,
                title=title)
        plt.show()
        logging.info('Plotted NDVI image')

    # Create classes and apply to NDVI results
    ndvi_class_bins = [-np.inf, 0, 0.1, 0.25, 0.4, np.inf]
    ndvi_class = np.digitize(data_ndvi, ndvi_class_bins)

    # Apply the nodata mask to the newly classified NDVI data
    ndvi_class = np.ma.masked_where(
        np.ma.getmask(data_ndvi), ndvi_class
    )
    np.unique(ndvi_class)

    # Define color map
    nbr_colors = ["gray", "y", "yellowgreen", "g", "darkgreen"]
    nbr_cmap = ListedColormap(nbr_colors)

    # Define class names
    ndvi_cat_names = [
        "No Vegetation",
        "Bare Area",
        "Low Vegetation",
        "Moderate Vegetation",
        "High Vegetation",
    ]
    legend_titles = ndvi_cat_names

    #plot ndvi classes
    fig, ax = plt.subplots(figsize=(28, 12))
    im = ax.imshow(ndvi_class, cmap=nbr_cmap)
    ep.draw_legend(im_ax=im, titles=legend_titles)
    ax.set_title(
        " Normalized Difference Vegetation Index (NDVI) Classes",
        fontsize=14,
    )
    ax.set_axis_off()

    #Auto adjust subplot to fit figure size
    plt.tight_layout()

    #save fig
    path_corr=path.replace("\\","/")+"_classes.png"
    plt.savefig(path_corr)
    logging.info(f'Saved ndvi classes figure {path_corr}')
    print(f'Saved figure {path_corr}')

    #plot figure
    if plot:
        print('Plotting NDVI classes image')
        plt.show()
    else:
        plt.close()

    return path_corr


import os
import time
import random
import math
import json
import functools

import config as cfg

def check_user_exists(exists_ok):
    def check_exists(func):
        @functools.wraps(func)
        def run(db, user, *args, **kwargs):
            if (user.id in db.users) == exists_ok:
                return func(db, user, *args, **kwargs)

            if exists_ok:
                raise Exception("User not found. Request denied.")
            raise Exception("User already exists. Request denied.")
        return run
    return check_exists

def admin_required(func):
    @functools.wraps(func)
    def run(self, *args, **kwargs):
        if self.curr_user and self.curr_user == self.db.admin:
            return func(self, *args, **kwargs)
        else:
            print("\nYou need to be logged in as admin to do this.", flush=True)
    return run

def login_required(func):
    @functools.wraps(func)
    def run(self, *args, **kwargs):
        if self.curr_user:
            return func(self, *args, **kwargs)
        else:
            print("\nYou need to be logged in to do this.", flush=True)
    return run

def logout_required(func):
    @functools.wraps(func)
    def run(self, *args, **kwargs):
        if not self.curr_user:
            return func(self, *args, **kwargs)
        else:
            print("\nYou need to be logged out to do this.", flush=True)
    return run

def delay_run(sec):
    def delay(func):
        @functools.wraps(func)
        def run(*args, **kwargs):
            time.sleep(sec)
            return func(*args, **kwargs)
        return run
    return delay

def loop_input_with_check(validation, input_msg=None, error_msg=None, exception=None):
    input_msg = "\n>>> Enter\n" if input_msg is None else input_msg
    error_msg = "\nInvalid field." if error_msg is None else error_msg
    exception = Exception if exception is None else exception

    while True:
        value = input(input_msg)
        if value == cfg.CANCEL_ACTION:
            raise InterruptedError("Cancelling request.")

        try:
            value = validation(value)
            break

        except exception:
            print(error_msg, flush=True)

        except:
            raise Exception
    return value