import asyncio, aiohttp
import re, os, time, smtplib, hashlib, urllib.request
import sqlite3 as sql
from random import randint
from email.message import EmailMessage
from Crypto.PublicKey import RSA
from Crypto.Cipher import PKCS1_OAEP
from config import *
# (Note: All variables not declared probably came from config.py)

#If email treated as spam:
 #https://support.google.com/mail/contact/bulk_send_new?rd=1

DB_CONN = sql.connect(DB_PATH)
KEY = RSA.import_key(DB_SECRET.encode("ascii"), SP) # Just you try and get it :D
BEN_10 = ["Heatblast", "Wildmutt", "Diamondhead", "XLR8", "Grey Matter",
          "Four Arms", "Stinkfly", "Ripjaws", "Upgrade", "Ghostfreak",
          "Cannonbolt", "Ditto", "Way Big", "Upchuck",
          "Wildvine", "Alien X", "Echo Echo", "Brainstorm", "Swampfire",
          "Humongousaur", "Jetray", "Big Chill", "Chromastone", "Goop",
          "Spidermonkey", "Rath", "Nanomech"]
SIKE = {'@':'a', '!': 'i', '1': 'i', '5': 's',
        '3': 'e', '0': 'o', 'l': 'i'}
CURSES = ["retard", "fuck", "shit", "ass",
          "pussy", "fucker", "dick", "nigger",
          "bitch", "nigg", "damn", "prick",
          "nigga", "hoe", "siut", "whore",
          "cunt", "dickhead", "isis", "taliban",
          "extremist", "terrorist"]

# Remove a line from a file based on value
def edit_file(file, value, exact=True):
    with open(file, 'r+', encoding="utf-8") as f:
        lines = f.readlines()
        f.seek(0); found = False
        if exact == True:
            for line in lines:
                line = line.strip('\n')
                if str(line).lower() != str(value).lower():
                    f.write(line + '\n')
                else:
                    found = True
        else:
            for line in lines:
                line = line.strip('\n')
                if str(value).lower() not in str(line).lower() :
                    f.write(line + '\n')
                else:
                    found = True
        f.truncate()
        return found

# Return 4-digit verification code string after sending email
def send_email(addr: str, gender='', test=False) -> str:
    sCode = f"{randint(0,9)}{randint(0,9)}{randint(0,9)}{randint(0,9)}"
    verify_link = f"{VERIFY_SITE}/verified/{sCode}/{gender}"
    verify_btn = f'VERIFY!'
    style_btn = """"""
    html = f"""{style_btn}
            Your verification link to join the chat is below:

            VERIFY!

{verify_link}

            Please click this link to join the {MSA} Discord. This link will expire in 15 minutes.
            """
    if not test:
        msg = EmailMessage()
        msg.set_content(html, subtype="html")
        msg["Subject"] = f"Verification Code for {MSA} MSA Discord"
        msg["From"] = EMAIL
        msg["To"] = addr
        with smtplib.SMTP_SSL("smtp.gmail.com", 465) as s:
            s.login(EMAIL,
                    APP_PASS)
            s.send_message(msg)
    else:
        print(sCode)
    return sCode

# SQL Query Function
def sqlite_query(query, args=(), one=False):
   cur = DB_CONN.cursor()
   cur = DB_CONN.execute(query, args); DB_CONN.commit()
   rv = [dict((cur.description[idx][0], value)
              for idx, value in enumerate(row)) for row in cur.fetchall()]
   return (rv[0] if rv else None) if one else rv

# Print SQL sample of records & return total records
def get_total_records() -> int:
   records = 0
   for row in sqlite_query(f"SELECT * FROM Links"):
      records += 1
   return records

# Encrypt msg string
def encrypt(msg):
    cipher = PKCS1_OAEP.new(KEY.publickey())
    cipher_text = cipher.encrypt(msg.encode())
    return cipher_text

# Decrypt cipher text
def decrypt(cipher_text):
    cipher = PKCS1_OAEP.new(KEY)
    decrypted_text = cipher.decrypt(cipher_text)
    return decrypted_text.decode()

# Return full name string based on email
def get_name(addr: str) -> str:
    sid = re.sub(r"@.+\.", '', str(addr))
    sid = sid.replace("edu", '')
    hashed_sid = hashlib.sha1(sid.encode()).hexdigest()
    query = f"SELECT full_name FROM Links WHERE sid=?"
    result = sqlite_query(query, (hashed_sid,), one=True)
    if result != None:
        full_name = result["full_name"]
        return decrypt(full_name)

# Check if message has curse word
def curse_check(msg: str) -> bool:
    msg = msg.replace('l', 'i')
    wordCheck = ''
    for i in range(len(msg)):
        char = msg[i]
        if char in SIKE:
            char = SIKE[char]
        wordCheck += char
        wordCheck = wordCheck.strip(' ')
        if wordCheck in CURSES:
            try:
                if msg[i+1] != char or msg[i+1] != ' ':
                    wordCheck = ''
                    continue
            except IndexError:
                pass
            return True
    for curse in CURSES:
        if re.search(fr"\b{curse}\b", msg):
            return True
    return False

# Return gender based on user
def check_gender(user):
    roles = user.roles
    for role in roles:
        if role.name == "Brother" or role.name == "Sister":
            return role.name

# Return true if user is admin or another role
def check_admin(msg, add_on=''):
    roles = msg.author.roles
    for role in roles:
        if role.name == "Admin" or "Shura" in role.name:
            return True
        if role.name == add_on:
            return True
    return False

# Return random Ben 10 alien
def ben_10(choice=''):
    choice = choice.strip(' '); alien_form = ''
    if choice == '':
        idx = randint(0,27)
        alien_form = BEN_10[idx]
    else:
        for alien in BEN_10:
            if alien.lower() in choice.lower():
                got = randint(1,3)
                if got == 1:
                    alien_form = alien; break
                else:
                    ignore = BEN_10.index(alien)
                    idx = randint(0,26)
                    temp = BEN_10[:ignore] + BEN_10[ignore+1:]
                    alien_form = temp[idx]; break
        if alien_form == '':
            idx = randint(0,27)
            alien_form = BEN_10[idx]
    return alien_form

# Retrieve role for those in waiting room
def get_sibling_role(member):
    roles = member.roles; ret = None
    for role in roles:
        if role.name == "Brothers Waiting Room":
            ret = ("Brother", role); break
        elif role.name == "Sisters Waiting Room":
            ret = ("Sister", role); break
    return ret

# Return sibling global object based on gender
def get_sibling(sibling):
    if sibling == "Brother":
        return BROTHERS
    else:
        return SISTERS

# Return announcement channel id while listening to announcements/events
def listen_announce(msg):
    if msg.channel.id == BROTHERS.announce:
        if "@everyone" in msg.content:
            return SISTERS.announce
    elif msg.channel.id == SISTERS.announce:
        if "@everyone" in msg.content:
            return BROTHERS.announce
    else:
        False

# Return role id based on emoji
def listen_role_reaction(emoji, channel):
    role_id = 0
    emoji = emoji.name.encode('unicode-escape')
    emote = re.search(r".+?\\", str(emoji).strip("b'\\"))
    if emote and str(emoji).lower().count('u') > 1:
        emoji = ("\\" + emote.group().strip('\\')).encode()
    for role_emoji in ROLE_EMOJIS:
        if "\\U" not in emoji.decode():
            if emoji.decode() in role_emoji:
                return ROLE_EMOJIS[role_emoji]
        if emoji == role_emoji.encode('unicode-escape'):
            return ROLE_EMOJIS[role_emoji]
    if len(emoji) == 11:
        emoji = emoji[1:].decode('unicode-escape')
    else:
        emoji = emoji.decode('unicode-escape')
    for role_select_channel in SPLIT_ROLES_EMOJIS:
        if role_select_channel == channel:
            return SPLIT_ROLES_EMOJIS[channel][emoji]
    return False

# Parse and return email & join type based on /verify request
def listen_verify(msg):
    if msg.channel.id == VERIFY_ID:
        if msg.content.startswith('/verify'):
            request = re.sub(r"/verify ", '', msg.content.lower())
            join_type = re.search(r"(brothers?|sis(tas?|ters?))", request) or ''
            if join_type:
                ucid = re.sub(fr"{join_type.group()}", '', request).strip(' ')
                if join_type.group()[0] == 'b':
                    join_type = "Brother"
                elif join_type.group()[0] == 's':
                    join_type = "Sister"
                else:
                    join_type = ''
                return ucid, join_type
            return (request, '')

# Listen for 4-digit code in #verify
def listen_code(msg):
    if msg.channel.id == VERIFY_ID:
        return re.search(r"^\d\d\d\d$", msg.content)

# Return sibling global object based on general channel_id
def in_general(channel_id):
    if channel_id == BROTHERS.general:
        return BROTHERS
    elif channel_id == SISTERS.general:
        return SISTERS
    else:
        return False

# Send Post Request
async def send_verify_post(data={}, test=False):
    if test:
        return '1'
    url = VERIFY_SITE + '/verify'
    data_encoded = urllib.parse.urlencode(data)
    data_encoded = data_encoded.encode("ascii")
    async with aiohttp.ClientSession() as session:
        async with session.post(url, data=data) as resp:
            result = await resp.text()
    return result

# Constantly check for changes in verify.txt
async def check_verify(record, msg, temp):
    while True:
        with open("verify.txt") as f:
            text = f.read()
            if not re.search(fr"{record}", text):
                break
        await asyncio.sleep(0)
    await msg.delete(); await temp.delete()


# -*- coding: utf-8 -*-
import urllib
import subprocess as s
import os
import argparse
import threading
import sys
import time
import re
import logging
import atexit
from urllib.request import Request, urlopen
import urllib.error
from xml.dom.minidom import parse

class CookieList:
	time = 0
	value = ""

path = os.path.expanduser("~/festeloApp/")

def exit_handler():
	logging.info('Script closing')

def createParser(): #Настройка аргументов
	parser = argparse.ArgumentParser()
	parser.add_argument('-n', '--new', nargs='+', help="Добавляет новый Cookie в базу данных")
	parser.add_argument('-r', '--run', nargs=1, help="Запускает вручную скрипт для выбранного Cookie")
	parser.add_argument('-i', '--info', nargs  ='+', help="Получает информацию об аккаунте из выбранного Cookie")
	parser.add_argument('-rm', '--remove', nargs=1, help="Удаляет Cookie из базы данных")
	parser.add_argument('-ls', '--list', default=False, action='store_true', help="Выводит список Cookie")

	return parser

def checkAkk(cookie): #Проверка Cookie аккаунта / получение данных об аккаунте
	try:
		#Открытие CSGO500 для получения данных об аккаунте
		f = urlopen(Request('https://csgo500.com/', headers={'User-Agent': 'Mozilla/5.0', 'Cookie': 'express.sid=' + cookie }))
	except urllib.error.HTTPError:
		print("Ошибка соединения")
		return 1
	result = f.read().decode('utf-8') # Записывание кода HTML страницы в переменную
	try:
		# Ищет на странице данные и записывает в переменные
		rewDate = re.findall('rewardDate = "(.*?) GMT', result)[0]
		nick = re.findall('
\n(.*?)\n
', result)[0]
		balance = re.findall('value = (.*?);', result)[0]
		return [nick, balance, rewDate] # Возвращает массив с полученными данными
	except IndexError:
		#Происходит если какие-то данные на странице не обнаружены.
		print("Ошибка авторизации, данные указаны верно?")
		return 2

def test(cookie, i):
	informing("Send request to CSGO500. Cookie#" + str(i), cookie, False)
	headers = {'User-agent': 'Mozilla/5.0', 'Cookie': 'express.sid=' + cookie}
	try:
		f = urlopen(Request('https://csgo500.com/', headers=headers)) #Открывает страницу для получения HTML
		res = f.read().decode("UTF-8") #Записывает данные в переменную
	except urllib.error.HTTPError:
		informing("Error with access to site! Check internet connection!", True)
		return 0
	token = re.findall('csrfToken = "(.*?)";', res) #Ищет CSRF токен на странице и записывает в переменную
	app_xml = parse(path + "/data.xml") #Открывает XML документ для чтения
	element = app_xml.childNodes[0].getElementsByTagName('Cookie')[i] #Находит нужный элемент Cookie по индексу i

	if '
' in res: #Происходит если на странице есть кнопка входа
		informing("Login error! Cookie #" + str(i), cookie, True)
		element.tagName = "OUTDATE"
	else:
		requestdata = {'_csrf' : token[0]} #Формирует csrf данные для POST запроса
		params_auth = urllib.parse.urlencode(requestdata)
		try:
			#Отправляет POST запрос для получения ежедневного бонуса
			data = urlopen(Request('https://csgo500.com/reward', data=params_auth.encode("UTF-8"), headers=headers))

		except urllib.HTTPError: #Происходит когда 1) Нет соединения. 2) Сервер отвечает кодом ошибки
			informing("The daily bonus has not yet come. Cookie #" + str(i), cookie, True)
			return 0
		element.removeAttribute("time") #Обновляет данные времени ежедневного бонуса в XML документе
		element.setAttribute("time", str(time.time()))
		informing("Task complete succesful! Cookie # " + str(i), cookie, False)
	res = open(path + "/data.xml", "w") #Открывает XML документ для записи
	app_xml.childNodes[0].writexml(res) #Записывает в него обновленные данные
	res.close() #Закрывает XML документ

def refrCookieList(mode = 0):
	try:
		dom = parse(path + "/data.xml") #Открывает для чтения XML документ
		custom = []
		i = 0
		if mode == 0: ElementName = 'Cookie'
		else: ElementName = 'OUTDATE'
		# Цикл для каждого элемента в XML документе с именем Cookie или OUTDATE
		for node in dom.childNodes[0].getElementsByTagName(ElementName):
			custom.append(CookieList()) #Добавляет к массиву новый объект CookieList и заполняет его
			custom[i].time = node.getAttribute("time")
			custom[i].value = node.firstChild.nodeValue
			i = i + 1 #Счетчик
		return custom
	except IOError:
		DataMake()

def DataMake():
	print("Первый запуск. Подготовка файлов...")
	try:
		os.makedirs(path) #Создает рабочую директорию
		f = open(path + "/data.xml", 'w') #Создает и открывает документ XML
		f.write("<data/>") #Записывает в него XML тег и закрывает его и файл
		f.close
		print("Файлы созданы, добавьте cookie командой 'febot -n COOKIE''")
		exit(0)
	except OSError:
		print("Ошибка при создании файлов, путь: " + path)
		exit(0)

def go():  #Таймер проверки надобности отправления запроса на ежедневный бонус
	custom = refrCookieList() #Обновляет данные с XML документа
	i = 0 #Счетчик
	for s in custom: #Проверка каждого элемента
		if float(s.time) > 0 and time.time() >= float(s.time) + 86400:
			test(s.value, i)
		i = i + 1
	threading.Timer(60.0, go).start() #Новый старт таймера

def informing(msg , addition = "", error = False):
	if not error:
		logging.info(msg + addition)
	else:
		logging.error(msg + addition)
	if(sys.platform == 'linux'):
		s.call(['notify-send', msg,'FesteloBot'])
	print(msg + addition)

#������ ���������: ������ ����������
atexit.register(exit_handler)
namespace = createParser().parse_args(sys.argv[1:]) #Парсер аргументов
checkArgs = False
logging.basicConfig(format = u'%(filename)s# %(levelname)-8s [%(asctime)s]  %(message)s', level = logging.DEBUG, filename = path+ "/logfile.log")
arguments = ""
for i in sys.argv[1:]:
	arguments += i + " "
logging.info( 'Script was launched. Args: ' + arguments)

if namespace.new != None:
	checkArgs = True
	try:
		print("\nПроцесс может занять некоторое время, в зависимости от скорости вашего интернет-соединения\n")
		okda = True
		data = []
		for s in namespace.new:
			a = checkAkk(s)
			if a != 1 and a != 2:
				data.append(a)
			else:
				print("Произошли ошибки в процессе добавления Cookie, измените запрос и/или повторите попытку позже.\nCookie: " + s)
				okda = False
				break
		if okda:
			app_xml = parse(path + "/data.xml")
			i = 0
			for s in namespace.new:
				newCo = app_xml.createElement("Cookie")
				a = time.strptime(data[i][2], "%a %b %d %Y %H:%M:%S")
				b = time.mktime(a)
				c = b - time.altzone + 86400
				newCo.setAttribute("time", str(c))
				newCo.appendChild(app_xml.createTextNode(s))
				app_xml.childNodes[0].appendChild(newCo)
				print("Добавлен COOKIE: Nick: {} | Balance: {} | Reward Date: {} UTC".format(data[i][0], data[i][1], data[i][2]))
				logging.info( u'Cookie ' +  s + 'was added #' + i)
				i = i + 1
			res = open(path + "/data.xml", "w")
			app_xml.childNodes[0].writexml(res)
			res.close()
			#if len(namespace.new) == 1:
			#	print("\nДобавлен Cookie:: " + namespace.new[0])
			#else:
			#	print("\nДобавлены Cookie:s: " + format(len(namespace.new)))
	except IOError:
		DataMake()

if namespace.run != None:
	checkArgs = True
	custom = refrCookieList()
	if int(namespace.run[0]) >= len(custom) or int(namespace.run[0]) < 0:
		print("Ашибка")
	else:
		test(custom[int(namespace.run[0])].value, int(namespace.run[0]))
		logging.info( u'Cookie ' +  int(namespace.run[0]) + 'run manually')

if namespace.info != None:
	checkArgs = True
	custom = refrCookieList()
	i = int(namespace.info[0])
	if i >= len(custom) or i < 0:
		print("Ашибка")
	else:
		a = checkAkk(custom[int(namespace.info[0])].value)
		if a != 1 and a != 2:
			print(namespace.info[0] + ": Nick: {} | Balance: {} | Reward Date: {} UTC".format(a[0], a[1], a[2]))

if namespace.remove != None:
	checkArgs = True
	dom = parse(path + "/data.xml")
	custom = refrCookieList()
	outdate = refrCookieList(1)
	if int(namespace.remove[0]) >= len(custom) + len(outdate) or int(namespace.remove[0]) < 0:
		print("Ашибка")
		exit(0)
	else:
		if(int(namespace.remove[0]) >= len(custom)):
			dom.childNodes[0].removeChild(dom.childNodes[0].getElementsByTagName('OUTDATE')[int(namespace.remove[0]) - len(custom)])
		else:
			dom.childNodes[0].removeChild(dom.childNodes[0].getElementsByTagName('Cookie')[int(namespace.remove[0])])
		logging.info( u'Cookie ' +  int(namespace.remove[0]) + 'was removed')
		res = open(path + "/data.xml", "w")
		dom.childNodes[0].writexml(res)
		res.close()

if namespace.list:
	checkArgs = True
	custom = refrCookieList()
	i = 0
	for s in custom:
		date = time.strftime("%a, %d %b %Y %H:%M:%S", time.gmtime(float(custom[i].time)))
		print("\n{}: Recent action: {} UTC\nValue: {}".format(i, date, custom[i].value))
		i = i + 1
	outdate = refrCookieList(1)
	for s in outdate:
		date = time.strftime("%a, %d %b %Y %H:%M:%S", time.gmtime(float(outdate[i - len(custom)].time)))
		print("\n{}: OUTDATE!! Recent action: {}(sste)\nValue: {}".format(i, date, outdate[i - len(custom)].value))
		i = i + 1
#�����
#path = "/home/" + os.getlogin() + "/festeloApp/"
#test()
if not checkArgs:
	logging.info( u'Script starting')
	go()


import click
from zipline.data.bundles import register
from zipline.data import bundles as bundles_module
from cn_zipline.bundles.tdx_bundle import tdx_bundle
import pandas as pd
import os
from cn_stock_holidays.zipline.default_calendar import register_calendar

@click.group()
def main():
    pass

@main.command()
@click.option(
    '-b',
    '--bundle',
    default='quantopian-quandl',
    metavar='BUNDLE-NAME',
    show_default=True,
    help='The data bundle to ingest.',
)
@click.option(
    '--assets-version',
    type=int,
    multiple=True,
    help='Version of the assets db to which to downgrade.',
)
@click.option(
    '--show-progress/--no-show-progress',
    default=True,
    help='Print progress information to the terminal.'
)
def ingest(bundle, assets_version, show_progress):
    if bundle == 'tdx':
        register('tdx', tdx_bundle, 'SHSZ')
    bundles_module.ingest(bundle,
                          os.environ,
                          pd.Timestamp.utcnow(),
                          assets_version,
                          show_progress,
                          )

def register_tdx():
    register('tdx', tdx_bundle, 'SHSZ')

if __name__ == '__main__':
    register('tdx1', tdx_bundle, 'SHSZ')
    bundles_module.ingest('tdx1',
                          os.environ,
                          pd.Timestamp.utcnow(),
                          show_progress=True,
                          )
    # main()


from urllib.request import urlopen
from urllib.error import URLError, HTTPError
import re
import sys
import os

from PyQt5.QtWidgets import QWidget, QMainWindow, \
    QAction, QFormLayout, QScrollArea, \
    QDesktopWidget, QGroupBox, QPushButton, QVBoxLayout, \
    QLabel, QGridLayout, QHBoxLayout, QLineEdit, QApplication, qApp, \
    QInputDialog, QWidgetItem
from PyQt5.QtGui import QPainter, QColor, QFont, QPixmap
from PyQt5.QtCore import Qt
from database import DataBase, download_data
from google import Google
from threading import Thread
from webbrowser import open_new

def download_img(url_image):
    name = re.findall('.*/(.+)', url_image)[0]
    name = name.split('?')[0]
    try:
        os.mkdir('UsersPhoto')
    except OSError:
        pass
    if name not in os.listdir(path='UsersPhoto\\'):
        try:
            with urlopen(url_image) as page:
                logo = page.read()
        except (URLError, HTTPError):
            return None
        image = open('UsersPhoto\\' + name, 'wb')
        image.write(logo)
        image.close()
    return 'UsersPhoto\\' + name

class MainWindow(QMainWindow):
    def __init__(self):
        super().__init__()
        self.db = DataBase()
        self.widget = Contacts(self.db)
        self.err_win = None
        self.init_ui()

    def init_ui(self):
        self.resize(400, 600)
        self.centre()
        self.setWindowTitle('export_contacts')
        self.setCentralWidget(self.widget)

        vk_action = QAction('&VK', self)
        vk_action.triggered.connect(self.widget.init_ui)

        fb_action = QAction('&Facebook', self)
        fb_action.triggered.connect(lambda: self.widget.init_ui())

        exit_action = QAction('&Exit', self)
        exit_action.triggered.connect(qApp.quit)

        th = Thread(target=self.widget.import_all_contacts)
        import_action = QAction('&Import all contacts', self)
        import_action.triggered.connect(lambda: th.start())

        search_action = QAction('&Find contact', self)
        search_action.triggered.connect(self.find_contact)

        menubar = self.menuBar()
        export = menubar.addMenu('&Export from')

        find_contact = menubar.addMenu('&Find Contact')
        find_contact.addAction(search_action)

        import_cont = menubar.addMenu('&Import all contacts')
        import_cont.addAction(import_action)

        ex = menubar.addMenu('&Exit')
        ex.addAction(exit_action)

        from_vk = export.addMenu('&VK')
        from_fb = export.addMenu('&Facebook')

        from_vk.addAction(vk_action)
        from_fb.addAction(fb_action)

        self.show()
        if self.db.db_exists():
            self.widget.init_ui()

    def find_contact(self):
        name, ok = QInputDialog.getText(self, 'Find contact',
                                        'Enter username')
        if ok:
            if self.widget.dic_wind.get(name) is not None:
                self.widget.dic_wind[name].init_ui()
            else:
                self.err_win = ErrorWindow(
                    'Contact ' + name + ' not found')
                self.err_win.init_ui()

    def centre(self):
        qr = self.frameGeometry()
        cp = QDesktopWidget().availableGeometry().center()
        qr.moveCenter(cp)
        self.move(qr.topLeft())

class Contacts(QWidget):
    def __init__(self, db):
        self.dic_wind = {}
        self.dic_button = {}
        self.dic_err_win = {}
        self.layout = None
        self.mygroupbox = QGroupBox('Contacts list')
        self.myform = QFormLayout()
        self.scroll = QScrollArea()
        self.db = db
        self.google_api = Google()
        super().__init__()

    def init_ui(self):
        sender = self.sender()
        error = None
        if sender is not None:
            if sender.text() == '&VK':
                if not self.db.db_exists():
                    error = self.db.create(download_data(sender.text()))
                else:
                    error = self.db.update_database(
                        download_data(sender.text()))
            if sender.text() == '&Facebook':
                if not self.db.db_exists():
                    error = self.db.create(download_data(sender.text()))
                else:
                    error = self.db.update_database(
                        download_data(sender.text()))

        if error is not None:
            self.dic_err_win[error] = ErrorWindow(error)
            self.dic_err_win[error].init_ui()
        else:
            for friend in self.db.get_list_users():
                button = self.create_button(friend)
                self.myform.addRow(button)

            self.mygroupbox.setLayout(self.myform)

            self.scroll.setWidget(self.mygroupbox)
            self.scroll.setWidgetResizable(True)
            self.scroll.setFixedHeight(600)
            self.layout = QVBoxLayout(self)
            self.layout.addWidget(self.scroll)

    def create_button(self, friend):
        name = friend
        button = QPushButton(name, self)
        self.dic_button[name] = button
        inf_friend = self.db.get_user_inf(friend)
        if inf_friend['picture'] != '':
            logo = inf_friend['picture']

        self.dic_wind[name] = Window(name, logo,
                                     inf_friend, self, self.google_api)
        button.clicked.connect(lambda: self.dic_wind[name].init_ui())
        return button

    def import_all_contacts(self):
        list_users = self.db.get_list_users()
        if len(list_users) != 0:
            open_new('https://contacts.google.com')
            for contact in list_users:
                contact_data = self.db.get_user_inf(contact)
                self.google_api.create_contact(
                    self.google_api.create_xml(contact_data))

    def redrawing(self):
        self.clear_window()
        self.init_ui()

    def clear_layout(self, layout):
        for i in range(layout.count()):
            if layout.itemAt(i) is not None:
                layout.itemAt(i).widget().setParent(None)

    def clear_window(self):
        for i in range(self.layout.count()):
            if self.layout.itemAt(i) is not None:
                if self.layout.itemAt(i).layout() is not None:
                    self.clear_layout(self.layout.itemAt(i).layout())
                    self.layout.itemAt(i).layout().setParent(None)
                if self.layout.itemAt(i).widget() is not None:
                    self.layout.itemAt(i).widget().setParent(None)

class Window(QWidget):
    def __init__(self, name_wind, url_logo, inf_friend,
                 parent_window, google_api):
        self.name_wind = name_wind
        self.url_logo = url_logo
        self.inf_friend = inf_friend
        self.titles = ['name:', 'bdate:', 'city:', 'country:',
                       'home_phone:', 'instagram:', 'skype:',
                       'email:', 'occupation:']
        self.edit_window = None
        self.parent_window = parent_window
        self.p_button = None
        self.grid = QGridLayout()
        self.form = QFormLayout()
        self.google_api = google_api
        super().__init__()

    def init_ui(self):
        self.clear_layout(self.grid)
        self.p_button = self.sender()
        vbox_title = QVBoxLayout()
        vbox_inf = QVBoxLayout()

        for title in self.titles:
            label = QLabel(title)
            vbox_title.addWidget(label)
            vbox_inf.addWidget(QLabel(self.inf_friend[title[:len(title) - 1]]))

        self.grid.addLayout(vbox_title, 0, 1)
        self.grid.addLayout(vbox_inf, 0, 2)

        photo = download_img(self.url_logo)
        logo = QLabel(self)
        if photo is not None:
            pixmap = QPixmap(photo)
            logo.setPixmap(pixmap)
        self.grid.addWidget(logo, 0, 0)

        edit_button = QPushButton('Edit', self)
        edit_button.clicked.connect(self.edit_inf_dialog)
        self.grid.addWidget(edit_button, 1, 0)

        th = Thread(target=self.import_contact)
        import_button = QPushButton('Import in Google Contacts', self)
        import_button.clicked.connect(lambda: th.start())
        self.grid.addWidget(import_button, 2, 0)

        delete_button = QPushButton('Delete contact', self)
        delete_button.clicked.connect(self.delete_contact)
        self.grid.addWidget(delete_button, 3, 0)

        self.setLayout(self.grid)
        self.setGeometry(300, 300, 700, 300)
        self.setWindowTitle(self.name_wind)
        self.show()

    def delete_contact(self):
        self.close()
        self.parent_window.dic_wind[self.name_wind].setParent(None)
        self.parent_window.dic_wind.pop(self.name_wind)
        self.parent_window.dic_button[self.name_wind].setParent(None)
        self.parent_window.dic_button.pop(self.name_wind)
        self.parent_window.db.delete_user(self.name_wind)

    def import_contact(self):
        self.google_api.create_contact(
            self.google_api.create_xml(self.inf_friend))
        open_new('https://contacts.google.com')

    def change_data(self, w_name, inf_friend):
        self.name_wind = w_name
        self.inf_friend = inf_friend

    def clear_layout(self, layout):
        for i in reversed(range(layout.count())):
            item = layout.itemAt(i)
            if isinstance(item, QWidgetItem):
                item.widget().close()
            else:
                self.clear_layout(item.layout())
            layout.removeItem(item)

    def repaint(self):
        self.clear_layout(self.grid)
        self.init_ui()

    def edit_inf_dialog(self):
        self.edit_window = EditWindow(self.titles, self.name_wind,
                                      self, self.parent_window, self.p_button)
        self.edit_window.init_ui()

class EditWindow(QWidget):
    def __init__(self, column, edit_user,
                 parent_window, base_widget, base_button):
        super().__init__()
        self.column = column
        self.edits = {}
        self.dic_edit_label = {}
        self.edit_user = edit_user
        self.parent_window = parent_window
        self.base_widget = base_widget
        self.base_button = base_button

    def init_ui(self):
        grid = QGridLayout()
        grid.setSpacing(10)
        hbox_button = QHBoxLayout()
        list_edits = []
        list_labels = []

        position_label = [(i, 0) for i in range(len(self.column))]
        position_edit = [(i, 1) for i in range(len(self.column))]

        for label, pos in zip(self.column, position_label):
            lb = QLabel(label)
            list_labels.append(lb)
            grid.addWidget(lb, *pos)

        for pos in position_edit:
            qle = QLineEdit(self)
            list_edits.append(qle)
            qle.textChanged[str].connect(self.on_changed)
            grid.addWidget(qle, *pos)

        self.dic_edit_label = \
            {x[0]: x[1] for x in zip(list_edits, list_labels)}

        ok_button = QPushButton('Ok', self)
        ok_button.clicked.connect(self.is_ok)
        cancel_button = QPushButton('Cancel', self)
        cancel_button.clicked.connect(self.close)

        hbox_button.addWidget(ok_button)
        hbox_button.addWidget(cancel_button)

        grid.addLayout(hbox_button, len(self.column) + 1, 0)

        self.setLayout(grid)

        self.setGeometry(900, 300, 350, 300)
        self.setWindowTitle('Edit information')
        self.show()

    def on_changed(self, text):
        self.edits[self.sender()] = text

    def is_ok(self):
        dic_change = {}
        for edit in self.dic_edit_label:
            if self.edits.get(edit) is not None:
                name = self.dic_edit_label[edit].text()
                dic_change[name[:len(name) - 1]] = self.edits[edit]
        self.edits = {}
        self.dic_edit_label = {}
        self.parent_window.close()
        self.create_new_parent_window(dic_change)
        self.close()

    def create_new_parent_window(self, dic_change):
        self.base_widget.db.update_user_inf(self.edit_user, dic_change)
        if dic_change.get('name') is not None:
            user_inf = self.base_widget.db.get_user_inf(dic_change['name'])
        else:
            user_inf = self.base_widget.db.get_user_inf(self.edit_user)
        self.parent_window.change_data(user_inf['name'], user_inf)
        self.parent_window.repaint()

class ErrorWindow(QWidget):
    def __init__(self, text_error):
        self.error = text_error
        super().__init__()

    def init_ui(self):
        self.setGeometry(300, 200, 300, 200)
        self.setWindowTitle('Error')
        self.show()

    def paintEvent(self, event):
        qp = QPainter()
        qp.begin(self)
        self.drawText(event, qp)
        qp.end()

    def drawText(self, event, qp):
        qp.setPen(QColor(0, 0, 0))
        qp.setFont(QFont('Decorative', 10))
        qp.drawText(event.rect(), Qt.AlignCenter, self.error)

if __name__ == '__main__':
    app = QApplication(sys.argv)
    widget = MainWindow()
    sys.exit(app.exec_())


import json
import boto3

def initDb():
    dynamodb = boto3.resource('dynamodb')
    return dynamodb

def deleteAllRecords(tableName):
    client = initDb()

    targetTable = client.Table(tableName)

    response = targetTable.scan()

    if 'Items' in response:
        items = response['Items']

        for i in items:
            targetTable.delete_item(Key={'id': i['id'], 'name': i['name']})
    else:
        items = ['No Items Deleted Or Error.']

    return items

def getAllRecords(tableName):
    client = initDb()

    targetTable = client.Table(tableName)

    response = targetTable.scan()

    if 'Items' in response:
        items = response['Items']
    else:
        items = ['No Items Found Or Error.']

    return items

def updateDb(tableName, obj):
    client = initDb()

    targetTable = client.Table(tableName)

    try:
        for item in obj:
            targetTable.put_item(Item=item)

        return {
            'statusCode': 200,
            'body': json.dumps('Succesfully inserted new montyly games!')
        }
    except:
        return {
            'statusCode': 400,
            'body': json.dumps('Error saving the free games')
        }


import socket
import configparser
import sys
import random
import time
import select
from functions import parse_ports, create_table, INFINITY, process_update, send_update, print_routing_table, extract_fields

def main(argv):

    loop = True

    try:
        configFile = argv[1]
    except (IndexError):
        return 'Missing file name'

    try:
        config = configparser.ConfigParser()
        config.read(configFile);

        router_id = int(config['ROUTER']['routerid'])
        input_ports_str = config['ROUTER']['inputports']
        outputs_str = config['ROUTER']['outputports']
        timeout = int(config['TIMER']['timeout'])
        update = int(config['TIMER']['update'])

        if router_id < 1 or router_id > 64000:
            raise ValueError('Router id is not in the specified range')

        input_ports = []
        input_ports_list = input_ports_str.split(',')
        for port in input_ports_list:
            input_port = int(port.strip())
            input_ports.append(input_port)

        parse_ports(input_ports)

        output_ports = []
        output_costs = []
        output_routerids = []
        outputs = outputs_str.split(',')
        for output in outputs:
            output_info = output.split('-')
            output_ports.append(int(output_info[0]))
            output_costs.append(int(output_info[1]))
            output_routerids.append(int(output_info[2]))

        parse_ports(output_ports)

        for i in range(0, len(output_ports)):
            for j in range(0, len(input_ports)):
                if output_ports[i] == input_ports[j]:
                    raise ValueError('A pair of input/output ports are equal')

        if timeout / update != 6:
            raise ValueError('Incorrect timeout/update timer ratio')

    except ValueError as e:
        print('ERROR: ' + str(e))
        sys.exit()

    in_socks = []
    for port in input_ports:
        host = socket.gethostname()
        sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        sock.bind((host, port))
        in_socks.append(sock)

    out_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)

    routing_table = create_table(output_costs, output_routerids, output_ports, timeout)
    print_routing_table(routing_table)

    while loop == True:

        time_before = time.time()

        #if triggered_updates_table.length > 0:
            #triggered_update(triggered_updates_table)

        offset = random.randrange(-200, 200, 1)
        update_timer = update * (1 + (offset / 1000))

        readables, writables, exceptionables = select.select(in_socks, [], [], update_timer)

        if readables:
            for sock in readables:
                print("Hello received packet thankyou")
                packet = sock.recv(2**16)
                entries, source_router = extract_fields(packet)
                process_update(routing_table, entries, source_router, output_ports)

        time_after = time.time()

        for i in range(0, len(routing_table)):
            entry = routing_table[i]

            if entry.timeout != 420:
                entry.timeout -= time_after - time_before
                if entry.timeout >= timeout:
                    entry.cost = INFINITY
                    entry.timeout = 420
                    entry.change_flag = 1
                    #triggered_update_table.append(entry)
            else:
                entry.garbage_timer -= time_after - time_before
                if entry.garbage_timer <= 0:
                    delete_route(entry)

        send_update(routing_table, output_ports, output_routerids, out_sock, router_id)
        print_routing_table(routing_table)

if __name__ == '__main__':
    sys.exit(main(sys.argv))


import os

from dotenv import load_dotenv

from controller import get_non_empty_matches
from mailer import send_mail
from movies_to_search import get_movies_to_search
from programs_fetcher import fetch_programs

def execute():
    load_dotenv()
    movies_to_search = get_movies_to_search()
    print(f"Trying to search movies: {movies_to_search}")
    programs = fetch_programs()
    matches_dict = get_non_empty_matches(movies_to_search, programs)
    if len(matches_dict) > 0:
        print(f"New movies found! Sending them by email. Detail: {matches_dict}")
        send_email_for_movies(matches_dict)
    else:
        print("No movie found on TV.")

def send_email_for_movies(matches):
    text = get_email_body(matches)
    send_mail(text, "[MoviesOnTvBot] New movies found on TV!", os.getenv("EMAIL_RECIPIENT"))

def get_email_body(matches):
    text = "Here is the list of movies found:"
    for movie, programmes in matches.items():
        text += f"\nMovie '{movie}':\n"
        for prog in programmes:
            text += f"{prog}\n"
    text += "\n Don't forget to update your movie list. Bye bye :)"
    return text

if __name__ == "__main__":
    execute()


from django.shortcuts import render, get_object_or_404, redirect
from .models import Loans, Clients
# from .forms import LoansForm
from django.contrib.auth.decorators import login_required
# import datetime
from datetime import datetime, timedelta
from .load_json import ConfigData
from kredyty.validators.loan_validation_hour import LoanValidationHour
from kredyty.validators.check_loan_validators import CheckLoanValidators
from kredyty.calculators.repayment_amount_calculator import RepaymentAmountCalculator
from django import forms

# REST_FRAMEWORK:
from rest_framework import viewsets
from django.contrib.auth.models import User
from .serializers import UserSerializer, LoansSerializer, ClientsSerializer, LoansSerializerPUT
from rest_framework.response import Response
from rest_framework.authentication import TokenAuthentication
from rest_framework.permissions import (
    IsAuthenticated, AllowAny, IsAuthenticatedOrReadOnly, IsAdminUser, DjangoModelPermissions)
from rest_framework import status

class UserViewSet(viewsets.ModelViewSet):
    queryset = User.objects.all()   # co chcemy wybierac z bd
    serializer_class = UserSerializer
    authentication_classes = (TokenAuthentication, )
    permission_classes = (IsAuthenticatedOrReadOnly, )

class LoansViewSet(viewsets.ModelViewSet):
    queryset = Loans.objects.all()
    serializer_class = LoansSerializer
    authentication_classes = (TokenAuthentication, )
    permission_classes = (IsAuthenticated, )

    def create(self, request, *args, **kwargs):
        if CheckLoanValidators().validate() == True:
            loan = request.data
            loan = loan.copy()
            loan['repayment_amount'] = RepaymentAmountCalculator(int(request.data["amount"]), float(ConfigData().get_data()['interest_rate'])).calculate()
            print(loan)
            serializer = LoansSerializer(data=loan)
            serializer.is_valid(raise_exception=True)    # validuje request
            self.perform_create(serializer)   # zapisuje do bazy
            headers = self.get_success_headers(serializer.data)
            return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers)
        else:
            pass

    def update(self, request, *args, **kwargs):
        partial = kwargs.pop('partial', False)
        instance = self.get_object()
        mod_data = request.data
        mod_data = mod_data.copy()
        mod_data['repayment_amount'] = RepaymentAmountCalculator(int(request.data["amount"]), float(ConfigData().get_data()['interest_rate'])).calculate()
        serializer = LoansSerializerPUT(instance, data=mod_data, partial=partial)
        serializer.is_valid(raise_exception=True)
        self.perform_update(serializer)

        return Response(serializer.data)

class ClientsViewSet(viewsets.ModelViewSet):
    queryset = Clients.objects.all()
    serializer_class = ClientsSerializer
    authentication_classes = (TokenAuthentication, )
    permission_classes = (IsAuthenticated, )

# PONIEZEJ FUNKCJE DO PRZEPISANIA

def loan_list(request):
    all_loans = Loans.objects.all()
    return render(request, 'kredyty.html', {'loans': all_loans})

@login_required
def new_loan(request):
    form = LoansForm(request.POST or None, request.FILES or None)
    form.fields['interest_rate'].widget = forms.HiddenInput()
    form.fields['repayment_amount'].widget = forms.HiddenInput()

    if form.is_valid():
        post = form.save(commit=False)
        repayment_amount = int(form['amount'].value()) * int(form['interest_rate'].value()) / 100 + int(form['amount'].value())
        post.repayment_amount = repayment_amount
        post.save
        form.save()
        return redirect(loan_list)

    if CheckLoanValidators().validate() == True:
        # return render(request, 'loan_form.html', {'form': form})
        return render(request, 'loan_form.html', {'form': form, 'interest_rate': ConfigData().get_data()['interest_rate']})
    else:
        return render(request, 'wrong_hour.html')

@login_required
def edit_loan(request, id):
    loan = get_object_or_404(Loans, pk=id)
    form = LoansForm(request.POST or None, request.FILES or None,
                     instance=loan)

    if form.is_valid():
        form.save()
        return redirect(loan_list)
    return render(request, 'loan_form.html', {'form': form})

@login_required
def delete_loan(request, id):
    loan = get_object_or_404(Loans, pk=id)

    if request.method == "POST":
        loan.delete()
        return redirect(loan_list)

    return render(request, 'confirm.html', {'loan': loan})


'''
业务: 开3个线程按照顺序打印ABC 10次
方案: 用semaphore
'''

import logging
import time
from threading import Semaphore, Thread, current_thread
from typing import List

import requests

session = requests.Session()

def req1(list: List, sem: Semaphore):
    while len(list) > 0:
        with sem:
            if len(list) <= 0:
                break
            session.get(f'https://deelay.me/100/http://httpbin.org/get?a=1')
            logging.info(f'{current_thread().name}-{list.pop(0)}')
        time.sleep(0.000000000000000001)

def main():
    sem = Semaphore(1)
    list = ['A', 'B', 'C'] * 10
    tasks = [Thread(target=req1, args=(list, sem)) for i in range(3)]
    for i in tasks:
        i.start()
    for i in tasks:
        i.join()

if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO)
    main()


from django.conf.urls import patterns, include, url
from django.conf.urls.static import static
from django.conf import settings

from django.contrib import admin
admin.autodiscover()

urlpatterns = patterns('',
	# Core
	url('', include('core.urls')),

	# Pronostic
	url('^pronostic/', include('pronostic.urls')),

    # Allauth
    url(r'^accounts/', include('allauth.urls')),

    # Admin
    url(r'^admin/', include(admin.site.urls)),
) + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)

urlpatterns += patterns('',
	(r'^media/(?P<path>.*)$', 'django.views.static.serve', {
	'document_root': settings.MEDIA_ROOT}))


import operator

from django.db import models
from django.utils.translation import gettext_lazy as _

class GeneratorQuerySet(models.QuerySet):
    def active(self):
        return self.filter(is_active=True)

    def as_generators(self):
        return [g.as_generator() for g in self.prefetch_related("conditions")]

class Generator(models.Model):
    SOURCES = {}

    is_active = models.BooleanField(_("is active"), default=True)
    group = models.CharField(_("group"), max_length=50)
    source = models.CharField(_("source"), max_length=50)

    objects = GeneratorQuerySet.as_manager()

    class Meta:
        verbose_name = _("generator")
        verbose_name_plural = _("generators")

    def __str__(self):
        return self.group

    def as_generator(self):
        source = self.SOURCES[self.source]
        return {
            "group": self.group,
            "conditions": [
                condition.as_condition() for condition in self.conditions.all()
            ],
            **source,
        }

class Condition(models.Model):
    OPERATORS = [
        (">", operator.gt),
        ("<", operator.lt),
        (">=", operator.ge),
        ("<=", operator.le),
        ("=", operator.eq),
        ("!=", operator.ne),
    ]

    generator = models.ForeignKey(
        Generator,
        on_delete=models.CASCADE,
        related_name="conditions",
        verbose_name=_("generator"),
    )
    variable = models.CharField(_("variable"), max_length=50)
    operator = models.CharField(
        _("operator"), max_length=10, choices=[(d, d) for d, op in OPERATORS]
    )
    value = models.IntegerField(_("value"))

    class Meta:
        ordering = ["pk"]
        verbose_name = _("condition")
        verbose_name_plural = _("conditions")

    def __str__(self):
        return "{o.variable} {o.operator} {o.value}".format(o=self)

    def as_condition(self):
        op = dict(self.OPERATORS)[self.operator]
        return {"variable": self.variable, "test": lambda v: op(v, self.value)}


import datetime
import django_filters

from django.shortcuts import render

from django_filters.rest_framework import FilterSet
from django_filters import DateRangeFilter, DateFilter, CharFilter, NumberFilter
from django.shortcuts import render

from django.views.generic import TemplateView
from django.views.generic.edit import CreateView

from django.contrib.messages.views import SuccessMessageMixin
from django.contrib.auth.mixins import LoginRequiredMixin, PermissionRequiredMixin
from django.urls import reverse_lazy
from django.http import HttpResponseRedirect

from rest_framework import viewsets, filters
from rest_framework.authentication import SessionAuthentication, BasicAuthentication
from rest_framework.permissions import IsAuthenticated

from django.db.models import Q

from .models import TimeSheet, PaymentMade, CashOut
from users.models import Clients
from .serializers import TimeSheetSerializer, PaymentMadeSerializer, CashOutSerializer

class AddVaTimeSheetView(SuccessMessageMixin, LoginRequiredMixin, CreateView):
    template_name = 'timesheet/add_timesheet.html'
    model = TimeSheet
    fields = [
        'company_tagging', 'shift_date', 'month_to_date', 'clients_full_name', 'client_full_name',
        'title_job_request', 'channel_job_requested', 'job_request_description',
        'time_in', 'time_out', 'total_items', 'additional_comments', 'hourly_rate_peso',
        'status', 'assigned_approval'
    ]
    success_message = "Successfully added a timesheet!"

    def form_valid(self, form):
        form.instance.assigned_approval = self.request.user.staffs
        return super().form_valid(form)

    def get_success_url(self):
        return reverse_lazy('timesheet:add_timesheet')

    def get_form(self, form_class=None):
        form = super().get_form(form_class)

        # make the 'client_name' field use a datalist to autocomplete
        form.fields['clients_full_name'].widget.attrs.update({'list':'full_names'})

        return form

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)

        # add 'clients_full_name' to the context to populate the datalist
        context['full_names'] = Clients.objects.all()

        return context

class VaTimeSheetView(TemplateView, LoginRequiredMixin):
    template_name = 'timesheet/va_timesheet.html'

class TimeSheetView(TemplateView, LoginRequiredMixin):
    template_name = 'timesheet/view_timesheet.html'

class TimeSheetMasterView(TemplateView, LoginRequiredMixin):
    template_name = 'timesheet/masterboard.html'

class TimeSheetFilter(FilterSet):
    shift_date__month = NumberFilter(
        field_name='shift_date', lookup_expr='month')
    shift_date__gte = DateFilter(field_name='shift_date', lookup_expr='gte')
    shift_date__lte = DateFilter(field_name='shift_date', lookup_expr='lte')
    assigned_approval__full_name = CharFilter(
        field_name='assigned_approval__full_name', lookup_expr='icontains')
    clients_full_name = CharFilter(
        field_name='clients_full_name__full_name', lookup_expr='icontains')

    class Meta:
        model = TimeSheet
        fields = (
            'shift_date__month',
            'assigned_approval__full_name',
            'clients_full_name',
            'shift_date__gte',
            'shift_date__lte',
        )

class PaymentMadeFilter(FilterSet):
    date__month = NumberFilter(field_name='date', lookup_expr='month')
    name = CharFilter(field_name='client_name__full_name',
                      lookup_expr='icontains')

    class Meta:
        model = PaymentMade
        fields = ('date__month', 'name')

class CashOutFilter(FilterSet):
    cash_date_release__month = NumberFilter(
        field_name='cash_date_release', lookup_expr='month')
    cash_date__gte = DateFilter(field_name='cash_date_release', lookup_expr='gte')
    cash_date__lte = DateFilter(field_name='cash_date_release', lookup_expr='lte')
    name = CharFilter(field_name='name__full_name', lookup_expr='icontains')

    class Meta:
        model = CashOut
        fields = ('cash_date_release__month', 'cash_date__gte',
                  'cash_date__lte', 'name')

class TimeSheetViewSet(viewsets.ModelViewSet):
    permission_classes = (IsAuthenticated,)
    serializer_class = TimeSheetSerializer
    filter_class = (TimeSheetFilter)

    def get_queryset(self):
        timesheet = TimeSheet.objects.all()
        current_year = datetime.date.today().year
        year = Q(shift_date__year=current_year)

        if self.request.user.is_client:
            client = Q(
                clients_full_name__full_name__icontains=self.request.user.clients.full_name)
            client_name = Q(client_full_name=self.request.user.clients.full_name)
            queryset = timesheet.filter(client | client_name & Q(
                admin_approval="Approved") & Q(status="Approved"))
            return queryset
        elif self.request.user.is_staffs:
            staff = Q(
                assigned_approval__full_name__icontains=self.request.user.staffs)
            qs = timesheet.filter(staff)
            return qs
        elif self.request.user.is_superuser:
            qs = TimeSheet.objects.all()
            return qs

class PaymentMadeViewSet(viewsets.ModelViewSet):
    permission_classes = (IsAuthenticated,)
    serializer_class = PaymentMadeSerializer
    filter_class = (PaymentMadeFilter)

    def get_queryset(self):
        if self.request.user.is_client:
            queryset = PaymentMade.objects.filter(Q(client_name__full_name=self.request.user.clients.full_name))
            return queryset
        elif self.request.user.is_superuser:
            qs = PaymentMade.objects.all()
            return qs

class CashOutViewSet(viewsets.ModelViewSet):
    permission_classes = (IsAuthenticated,)
    serializer_class = CashOutSerializer
    filter_class = (CashOutFilter)

    def get_queryset(self):
        if self.request.user.is_staffs:
            queryset = CashOut.objects.filter(Q(name__full_name=self.request.user.staffs.full_name))
            return queryset
        elif self.request.user.is_superuser:
            qs = CashOut.objects.all()
            return qs

def client_company_name(request):
    client_name = Clients.objects.all()
    return render(request, 'timesheet/add_timesheet.html',  {"client_names": client_name})


import numpy as np
import matplotlib.pyplot as plt
import pathlib
from pathlib import Path
import os
import random

import PIL
import PIL.Image
import tensorflow as tf
import sys
from tensorflow import keras
from tensorflow.keras import layers
from tensorflow.keras import models
from tensorflow.keras.models import Sequential
from tensorflow.keras.callbacks import CSVLogger
import pandas as pd
from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay
# import tensorflow_datasets as tfds
import seaborn as sns
import glob
print(tf.__version__)

object_category_names = sorted(os.listdir('train'))
object_categories = {}
for i, name in enumerate(object_category_names):
    object_categories[i] = name

#result = object_categories.items()
#data = list(result)

#object_labels_two = np.array(data)
object_labels_two = np.delete(np.array(list(object_categories.items())),0,1)
object_labels = object_labels_two.reshape(len(object_categories),)

print(object_labels)

def display_sample(sample_images, sample_labels, sample_predictions=None, num_rows=8, num_cols=8,
                   plot_title=None, fig_size=None):
    """ display a random selection of images & corresponding labels, optionally with predictions
        The display is laid out in a grid of num_rows x num_col cells
        If sample_predictions are provided, then each cell's title displays the prediction
        (if it matches actual) or actual/prediction if there is a mismatch
    """
    from PIL import Image
    import seaborn as sns
    assert len(sample_images) == num_rows * num_cols

    # a dict to help encode/decode the labels
    global object_categories

    with sns.axes_style("whitegrid"):
        sns.set_context("notebook", font_scale=1.1)
        sns.set_style({"font.sans-serif": ["Verdana", "Arial", "Calibri", "DejaVu Sans"]})

        f, ax = plt.subplots(num_rows, num_cols, figsize=((14, 12) if fig_size is None else fig_size),
            gridspec_kw={"wspace": 0.02, "hspace": 0.25}, squeeze=True)
        #fig = ax[0].get_figure()
        f.tight_layout()
        f.subplots_adjust(top=0.93)

        for r in range(num_rows):
            for c in range(num_cols):
                image_index = r * num_cols + c
                ax[r, c].axis("off")
                # show selected image
                pil_image = sample_images[image_index] #Image.fromarray(sample_images[image_index])
                ax[r, c].imshow(pil_image.astype('uint8'))

                if sample_predictions is None:
                    # show the actual labels in the cell title
                    title = ax[r, c].set_title("%s" % object_categories[sample_labels[image_index]])
                else:
                    # else check if prediction matches actual value
                    true_label = sample_labels[image_index]
                    pred_label = sample_predictions[image_index]
                    prediction_matches_true = (sample_labels[image_index] == sample_predictions[image_index])
                    if prediction_matches_true:
                        # if actual == prediction, cell title is prediction shown in green font
                        title = object_categories[true_label]
                        title_color = 'g'
                    else:
                        # if actual != prediction, cell title is actua/prediction in red font
                        title = '%s/%s' % (object_categories[true_label][:5], object_categories[pred_label][:5])
                        title_color = 'r'
                    # display cell title
                    title = ax[r, c].set_title(title)
                    plt.setp(title, color=title_color)
        # set plot title, if one specified
        if plot_title is not None:
            f.suptitle(plot_title)

        plt.show()
        plt.close()

#args = -1
batch_size = 128
img_height = 200
img_width = 200

train_ds = tf.keras.preprocessing.image_dataset_from_directory(
	Path(os.getcwd() + "\\train"),
	labels="inferred",
	color_mode='rgb',
	validation_split=0.2,
	subset="training",
	seed=123,
	image_size=(img_height, img_width),
	batch_size=batch_size)

val_ds = tf.keras.preprocessing.image_dataset_from_directory(
	Path(os.getcwd() + "\\train"),
	labels="inferred",
	color_mode='rgb',
	validation_split=0.2,
	subset="validation",
	seed=123,
	image_size=(img_height, img_width),
	batch_size=batch_size)

test_ds = tf.keras.preprocessing.image_dataset_from_directory(
	Path(os.getcwd() + "\\test"),
	labels="inferred",
	color_mode='rgb',
	seed=123,
	shuffle=True,
	image_size=(img_height, img_width),
	batch_size=batch_size)

train_steps = int(train_ds.cardinality() - 1)
validation_steps = int(val_ds.cardinality() - 1)
test_steps = int(test_ds.cardinality() - 1)

print('Training set size %d, step-size %d' % (train_steps*batch_size, train_steps))
print('Validation set size %d, step-size %d' % (validation_steps*batch_size, validation_steps))
print('Test set size %d, step-size %d' % (test_steps*batch_size , test_steps))
class_names = train_ds.class_names
train_ds = train_ds.repeat()
val_ds = val_ds.repeat()

normalization_layer = tf.keras.layers.experimental.preprocessing.Rescaling(1./255)

normalized_train_ds = train_ds.map(lambda x, y: (normalization_layer(x), y))

images, labels = iter(normalized_train_ds).next()
#print(labels)
nd_labels = labels.numpy()
nd_images = images.numpy()
#labels = np.argmax(labels, axis=1)

display_sample(nd_images[:16], nd_labels[:16], num_rows=4, num_cols=4, plot_title='Sample Training Data', fig_size=(16,20))

if Path.exists(Path(os.getcwd() + "\\saved_model")):
	print("./saved_model existiert bereits.")
	model = tf.keras.models.load_model('Q:/rgbd-data/RGBD multimodal/saved_model')

	print("Modell geladen.")

else:

	print(class_names)
	normalization_layer = tf.keras.layers.experimental.preprocessing.Rescaling(1./255)

	#normalized_ds = train_ds.map(lambda x, y: (normalization_layer(x), y))
	#image_batch, labels_batch = next(iter(normalized_ds))
	#first_image = image_batch[0]
	# Notice the pixels values are now in `[0,1]`.
	# print(np.min(first_image), np.max(first_image))
	#
	# AUTOTUNE = tf.data.experimental.AUTOTUNE
	#
	# train_ds = train_ds.cache().prefetch(buffer_size=AUTOTUNE)
	# val_ds = val_ds.cache().prefetch(buffer_size=AUTOTUNE)

	num_classes = len(class_names)
	# num_classes = 5
	#factor = 0.5
	#regularizer = contrib.layers.l2_regularizer(scale=0.1)
	model = tf.keras.Sequential([
		layers.experimental.preprocessing.Rescaling(1./255),
		#layers.Conv2D(16*factor, 3, activation='relu'),
		#layers.MaxPooling2D(),
		#layers.Activation('sigmoid'),
		layers.Conv2D(32, 3, activation='relu'),
		layers.MaxPooling2D(),
		#layers.Activation('sigmoid'),
		layers.Conv2D(64, 3, activation='relu'),
		#layers.Activation('sigmoid'),
		layers.MaxPooling2D(),
		layers.Conv2D(128, 3, activation='relu'),
		layers.MaxPooling2D(),
		layers.Conv2D(256, 3, activation='relu'),
		layers.MaxPooling2D(),
		layers.Dropout(0.10),
		layers.Dense(1024, activation='relu', kernel_regularizer="l2"),
		layers.MaxPooling2D(),
		layers.Dropout(0.10),
		#layers.Dropout(.2),
		#layers.Activation('sigmoid'),
		#layers.Conv2D(128, 3, activation='relu'),
		#layers.Conv2D(512*factor, 3, activation='relu'),
		#layers.MaxPooling2D(),
		#layers.Activation('sigmoid'),
		layers.Flatten(),
		#layers.Flatten(),
		layers.Dense(1024, activation='relu', kernel_regularizer="l2"),
		layers.Dropout(0.10),
		#layers.Softmax(),
		layers.Dense(num_classes)
	])

	model.compile(
		optimizer='adam',
		loss=tf.losses.SparseCategoricalCrossentropy(from_logits=True),
		metrics=['accuracy'])

	epochs = 15
	faster_steps = train_steps // 4
	csv_logger = CSVLogger('training.log', separator=',', append=False)
	#csv_logger_test = CSVLogger('test.log', separator=',', append=False)

	history = model.fit(
		train_ds,
		validation_data=val_ds,
		callbacks=[csv_logger],
		epochs=epochs,
		steps_per_epoch=faster_steps,
		validation_steps=validation_steps

	)

	model.save(
		'Q:/rgbd-data/RGBD multimodal/saved_model',
		overwrite=True,
		include_optimizer=True,
		save_format=None,
		signatures=None,
		options=None,
		save_traces=True,
	)

if Path.exists(Path(os.getcwd() + "\\saved_model")):
	log_data = pd.read_csv('training.log', sep=',', engine='python')
	history = log_data
	acc = history['accuracy']
	val_acc = history['val_accuracy']
	loss = history['loss']
	val_loss = history['val_loss']

else:
	history = model.history

	acc = history.history['accuracy']
	val_acc = history.history['val_accuracy']

	loss = history.history['loss']
	val_loss = history.history['val_loss']

plt.figure(figsize=(8, 8))
plt.subplot(1, 2, 1)
# plt.plot(epochs_range, acc, label='Training Accuracy')
# plt.plot(epochs_range, val_acc, label='Validation Accuracy')
plt.plot(acc, label='Training Accuracy')
plt.plot(val_acc, label='Validation Accuracy')
plt.legend(loc='lower right')
plt.title('Training and Validation Accuracy')

plt.subplot(1, 2, 2)
# plt.plot(epochs_range, loss, label='Training Loss')
# plt.plot(epochs_range, val_loss, label='Validation Loss')
plt.plot(loss, label='Training Loss')
plt.plot(val_loss, label='Validation Loss')
plt.legend(loc='upper right')
plt.title('Training and Validation Loss')
plt.show()

model.summary()
plt.clf()

all_predictions, all_labels = [], []
#all_predictions = np.empty(test_steps * batch_size, dtype=np.int)
#all_labels = np.empty(test_steps * batch_size, dtype=np.int)

print("Total test batches to predict: ",test_steps)

for i in range(test_steps):
	print("Testing batch nr:",i,"out of",test_steps)
	images, labels = iter(test_ds).next()
	predict_labels = model.predict(images)
	predict_labels = np.argmax(predict_labels, axis=1)
	nd_labels = labels.numpy()
	nd_images = images.numpy()

	all_predictions.extend(predict_labels)
	all_labels.extend(nd_labels)

all_labels = np.array(all_labels)
all_predictions = np.array(all_predictions)
#np.savetxt("test_labels.csv", all_labels, delimiter=',')test_steps
#np.savetxt("test_predictions.csv", all_predictions, delimiter=',')

def int_label_to_string(x):
  return object_labels[x]

print("\n Confusion Matrix:")
cm = confusion_matrix(int_label_to_string(all_labels),int_label_to_string(all_predictions), labels=object_labels,normalize='true')
np.savetxt("cm.csv", cm, delimiter=",")
disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=object_labels)
#plt.figure(figsize=(10, 10))
disp.plot(xticks_rotation='vertical')
for labels in disp.text_.ravel():
    labels.set_fontsize(6)

plt.savefig('confusion_matrix.png',dpi=300)
#plt.show()

#number_of_samples = np.zeros((len(object_categories),), dtype=np.uint64)
#wrong_labels = np.zeros((len(object_categories),), dtype=np.uint64)
#accuracies = np.zeros((len(object_categories),), dtype=np.uint64)
#for i in range(len(all_labels)):
#	number_of_samples[all_labels[i]] += 1
#	if all_labels[i] != all_predictions[i]:
#		wrong_labels[all_labels[i]] +=1

#print("Per object accuracy:")
for i in range(len(object_categories)):
#	correct_amount = number_of_samples[i] - wrong_labels[i]
#	accuracies[i] = correct_amount / number_of_samples[i]
#	print(object_categories[i]," accuracy: ", correct_amount / number_of_samples[i])
	print(object_categories[i]," accuracy: ", cm[i,i])

correct_pred_count = (all_labels == all_predictions).sum()
test_acc = correct_pred_count / len(all_labels)
print('\n We got %d of %d correct (or %.3f accuracy)' % (correct_pred_count, len(all_labels), test_acc))
#loss, acc = model.evaluate(test_ds, steps=test_steps, verbose=1)
#print('Testing data   -> loss: %.3f, acc: %.3f' % (loss, acc))

images, labels = iter(test_ds).next()

#print(labels)
nd_labels = labels.numpy()
nd_images = images.numpy()
#labels = np.argmax(labels, axis=1)

#display_sample(nd_images[:16], nd_labels[:16], num_rows=4, num_cols=4, plot_title='Sample Training Data', fig_size=(16,20))
predict_labels = model.predict(images)
predict_labels = np.argmax(predict_labels, axis=1)

#print("\n Actual label index:\n", nd_labels)
#print("\n Predicted label index:\n", predict_labels)

# display sample predictions

display_sample(nd_images[:64], nd_labels[:64], sample_predictions=predict_labels[:64], num_rows=8, num_cols=8,
               plot_title='Sample Predictions on Test Dataset', fig_size=(18,20))

#normalized_test_ds = test_ds.map(lambda x, y: (normalization_layer(x), y))
#images, labels = iter(normalized_test_ds).next()
#images, labels = iter(normalized_test_ds).next()

#score = model.evaluate(test_ds,verbose=1)
#print(score)
print()
loss, acc = model.evaluate(train_ds, steps=train_steps, verbose=1)
print('Training data  -> loss: %.3f, acc: %.3f' % (loss, acc))
loss, acc = model.evaluate(val_ds, steps=validation_steps, verbose=1)
print('Cross-val data -> loss: %.3f, acc: %.3f' % (loss, acc))
loss, acc = model.evaluate(test_ds, steps=test_steps, verbose=1)
print('Testing data   -> loss: %.3f, acc: %.3f' % (loss, acc))


#!/usr/bin/env python
# coding: utf-8

from bs4 import BeautifulSoup
import numpy as np
import pandas as pd
import nltk
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.svm import LinearSVC
from sklearn.pipeline import Pipeline
import string
from nltk.stem import PorterStemmer
import re
import spacy
import nltk
from io import StringIO
from nltk.tokenize import word_tokenize

ps = PorterStemmer()

#Stemming Function
def stemming(review):
    stem_review = [ps.stem(word) for word in review]
    return ' '.join(stem_review)

#Fuction to remove HTML
def removeHtml(review):
    review = re.compile(r'<[^>]+>').sub('', review)
    return review.lower().strip()

#Function to remove punctuation
def remove_punctuation(review):
    punctuationfree="".join([i for i in review if i not in string.punctuation])
    return punctuationfree

#Tokenizer
def tokenization(review):
    #tokens = re.split('W+',text)
    tokens = word_tokenize(review)
    tokens = [word for word in tokens if not word in stop_words]
    return tokens

#Finds similarities between training vectors and test vector
def findSimilarities(train_vect, test_vect):
    cosineSimilarities = np.dot(test_vect, np.transpose(train_vect)) #dot product of the 2 vectors
    similarities = cosineSimilarities.toarray()
    return similarities

def findNeighbors(similarities, k):
    return np.argsort(-similarities)[:k]

def predict(nearestNeighbors, sentiments):
    """Calculates the count of positive and negative reviews from similarity vector.
        If positive reviews are more, then the test review is positive and vice-versa"""
    #print(labels)

    positiveReviews = 0
    negativeReviews = 0
    for neighbor in nearestNeighbors:
        #print(neighbor)
        if int(sentiments[neighbor]) == 1:
            positiveReviews += 1
        else:
            negativeReviews += 1
    if positiveReviews > negativeReviews:
        return 1
    else:
        return -1

nlp = spacy.load('en_core_web_sm')

#Set the stopwords
stop_words = set(nltk.corpus.stopwords.words("english"))

def remove_stopwords(review):
    output= [i for i in review if i not in stop_words]
    return output

#Read in training file
train_data = pd.read_csv('./train_file.txt', sep='\t', header = None, skiprows=[0])
t = open("./test_file.txt", "r")
test_data = t.readlines()
test_data = pd.DataFrame(test_data)
print("Reading in data...\n")

#Set the column headers
column_names = ['sentiment', 'review']
train_data.columns = column_names
test_data.columns = ['review']
print("Preprocessing training and testing files...\n")

#Remove HTML
train_data['review'] = train_data['review'].apply(lambda x:removeHtml(x))
test_data['review'] = test_data['review'].apply(lambda x:removeHtml(x))

#Remove puntuation
train_data['review'] = train_data['review'].apply(lambda x:remove_punctuation(x))
test_data['review'] = test_data['review'].apply(lambda x:remove_punctuation(x))

#Tokenization
train_data['review'] = train_data['review'].apply(lambda x:tokenization(x))
test_data['review'] = test_data['review'].apply(lambda x:tokenization(x))

#Stemming
train_data['review'] = train_data['review'].apply(lambda x:stemming(x))
test_data['review'] = test_data['review'].apply(lambda x:stemming(x))

#set
y = train_data['sentiment']
x = train_data['review']
test_reviews = test_data['review']

#split data into training and testing sets
#from sklearn.model_selection import train_test_split

#x_train, x_test, y_train, y_test = train_test_split(x, y,  test_size = 0.25, random_state = 0)
#print("Train: ", x_train.shape, y_train.shape, "Test: ", (x_test.shape, y_test.shape))

print("Creating TF-IDF vectors...\n")
#Vectorization
vectorizer = TfidfVectorizer()
tf_x_train = vectorizer.fit_transform(x)
tf_x_test = vectorizer.transform(test_reviews)

#print(tf_x_train, tf_x_test)
#print(vectorizer.idf_)
print("Creating similarity model...\n")

#Find Similarities
similarities = findSimilarities(tf_x_train, tf_x_test)

k = 250
test_sentiments = list()

print("Predicting Sentiments...\n")
for similarity in similarities:
    knn = findNeighbors(similarity, k)
    prediction = predict(knn, y)

    #To write to the list as +1 instead of just a 1 for positive reviews
    if prediction == 1:
        test_sentiments.append('1')
    else:
        test_sentiments.append('-1')

#Write the result to a txt file
output = open('format.txt', 'w')

output.writelines( "%s\n" % item for item in test_sentiments )

output.close()
print("Predictions complete")


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
from django.conf import settings

class Migration(migrations.Migration):

    dependencies = [
        ('idc', '0001_initial'),
        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
    ]

    operations = [
        migrations.CreateModel(
            name='Backup',
            fields=[
                ('id', models.AutoField(serialize=False, primary_key=True)),
                ('name', models.CharField(max_length=64, verbose_name='Name')),
                ('backup_type', models.IntegerField(default=1, verbose_name='Back Type', choices=[(1, 'BACKUP_TYPE_FULL'), (2, 'BACKUP_TYPE_INCREMENT')])),
                ('volumes', models.CharField(default=None, max_length=255, null=True, verbose_name='Volumes', blank=True)),
                ('instance', models.IntegerField(default=None, null=True, verbose_name='Instance', blank=True)),
                ('create_date', models.DateTimeField(auto_now_add=True, verbose_name='Create Date')),
                ('delete_date', models.DateTimeField(auto_now=True, verbose_name='Delete Date')),
                ('deleted', models.BooleanField(default=False, verbose_name='Deleted')),
                ('status', models.IntegerField(default=0, verbose_name='Status', choices=[(0, 'Backup Waiting'), (1, 'Backup Available'), (2, 'Backup Error'), (9, 'Backup Deleted'), (10, 'Backup Backuping'), (11, 'Backup Restoring'), (20, 'Backup Pending Delete'), (21, 'Backup Pending Restore'), (19, 'Backup Deleting')])),
                ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL)),
                ('user_data_center', models.ForeignKey(to='idc.UserDataCenter')),
            ],
            options={
                'ordering': ['-create_date'],
                'db_table': 'backup',
                'verbose_name': 'Backup',
                'verbose_name_plural': 'Backup',
            },
            bases=(models.Model,),
        ),
        migrations.CreateModel(
            name='BackupItem',
            fields=[
                ('id', models.AutoField(serialize=False, primary_key=True)),
                ('resource_id', models.IntegerField(default=0, null=True, verbose_name='Resource ID', blank=True)),
                ('resource_uuid', models.CharField(max_length=64, null=True, verbose_name='Backend UUID', blank=True)),
                ('resource_type', models.CharField(max_length=64, null=True, verbose_name='Resource', blank=True)),
                ('resource_name', models.CharField(max_length=64, null=True, verbose_name='Resource Name', blank=True)),
                ('resource_size', models.IntegerField(default=0, null=True, verbose_name='Size(GB)', blank=True)),
                ('rbd_image', models.CharField(default=None, max_length=256, null=True, verbose_name='RBD Image', blank=True)),
                ('create_date', models.DateTimeField(auto_now_add=True, verbose_name='Create Date')),
                ('delete_date', models.DateTimeField(auto_now=True, verbose_name='Delete Date')),
                ('deleted', models.BooleanField(default=False, verbose_name='Deleted')),
                ('status', models.IntegerField(default=0, verbose_name='Status', choices=[(0, 'Backup Waiting'), (1, 'Backup Available'), (2, 'Backup Error'), (9, 'Backup Deleted'), (10, 'Backup Backuping'), (11, 'Backup Restoring'), (20, 'Backup Pending Delete'), (21, 'Backup Pending Restore'), (19, 'Backup Deleting')])),
                ('backup', models.ForeignKey(related_name='items', to='backup.Backup')),
                ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL)),
                ('user_data_center', models.ForeignKey(to='idc.UserDataCenter')),
            ],
            options={
                'ordering': ['-create_date'],
                'db_table': 'backup_item',
                'verbose_name': 'Backup Item',
                'verbose_name_plural': 'Backup Item',
            },
            bases=(models.Model,),
        ),
    ]


import os
import sys
import warnings
from argparse import ArgumentParser
from functools import partial
from glob import glob
from multiprocessing import Pool
from typing import List, Callable, Tuple, Union

import cv2
import numpy as np
import pandas as pd
from scipy.stats import kurtosis, skew

warnings.filterwarnings("ignore")

EXTERNAL_UTILS_LIB = "../alaska_utils"
sys.path.append(EXTERNAL_UTILS_LIB)

from alaska_utils import safe_mkdir
from alaska_utils import configure_arguments
from alaska_utils import parse_image_to_dir_basename

def process_image(image_file_path: str, functions: Tuple[str, Callable], ) -> Union[pd.Series, pd.DataFrame]:
    image = cv2.imread(image_file_path, cv2.IMREAD_COLOR)
    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)
    tmp = {name: [func(image[:, :, i].flatten()) for i in range(image.shape[-1])] for name, func in functions}
    for name, func in functions:
        tmp[name].append(func(image.flatten()))
    tmp = pd.DataFrame(tmp).stack().swaplevel()
    return tmp

def main(args: ArgumentParser):
    for dir_path in [args.output_dir, args.cached_dir, args.model_dir]:
        safe_mkdir(dir_path)

    args = configure_arguments(args)
    file_path: str = args.file_path_images_stats
    if os.path.exists(file_path) and not args.refresh:
        print(f"{file_path} exists, skip generates meta info")
        return False

    # process
    list_of_functions: List[Tuple[str, Callable]] = [
        ("mean", np.mean), ("std", np.std), ("kurt", kurtosis), ("skew", skew), ]
    # file path
    list_all_images: List[str] = list(glob(os.path.join(args.data_dir, "*", "*.jpg")))
    if args.debug:
        list_all_images = list_all_images[:100]

    with Pool(processes=args.n_jobs) as p:
        func = partial(process_image, functions=list_of_functions)
        df = pd.concat(list(p.map(func, list_all_images)), axis=1).astype(np.float32).T
        df.columns = [f"{i}_{m}" for m, i in df.columns]

    df_train = parse_image_to_dir_basename(args, list_all_images, column="file_path")

    # compose return dataframe
    image, kind = args.shared_indices
    df[image] = df_train[image].tolist()
    df[kind] = df_train[kind].tolist()
    df.sort_values(image, inplace=True)
    df.set_index(args.shared_indices, inplace=True)

    df.to_parquet(file_path)
    print(f"Save stats to: {file_path}\n{df.describe().T}")
    return

if "__main__" == __name__:
    #
    default_output_dir: str = "../input/alaska2-image-steganalysis-output/"
    default_cached_dir: str = "../input/alaska2-image-steganalysis-cached-data/"
    default_meta_dir: str = "../input/alaska2-image-steganalysis-image-quality/"
    default_model_dir: str = "../input/alaska2-image-steganalysis-models/"
    default_data_dir: str = "../input/alaska2-image-steganalysis/"
    #
    default_n_jobs: int = 8
    default_init_seed: int = 42

    parser = ArgumentParser()
    parser.add_argument("--output-dir", type=str, default=default_output_dir, help="folder for output")
    parser.add_argument("--cached-dir", type=str, default=default_cached_dir, help="folder for cached data")
    parser.add_argument("--meta-dir", type=str, default=default_meta_dir, help="folder for meta data")
    parser.add_argument("--model-dir", type=str, default=default_model_dir, help="folder for models")
    parser.add_argument("--data-dir", type=str, default=default_data_dir, help="folder for data")
    #
    parser.add_argument("--refresh", action="store_true", default=False, help="refresh cached data")
    parser.add_argument("--n-jobs", type=int, default=default_n_jobs, help="num worker")
    # debug
    parser.add_argument("--debug", action="store_true", default=False, help="debug")
    args = parser.parse_args()

    # start program
    main(args)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# pylint: disable=C0103
"""
Discourse relation types model/task.
"""
__author__ = "GW [http://gw.tnode.com/] <gw.2015@tnode.com>"
__license__ = "GPLv3+"

import os, sys
sys.path.append(os.path.dirname(os.path.dirname(__file__)))
import numpy as np
from keras.layers.core import Activation, TimeDistributedDense, RepeatVector, Permute
from keras.layers.recurrent import GRU

from common import build_index
from conll16st.relations import tag_to_rtsip, filter_tags

### Model

def rel_types_model(model, ins, max_len, embedding_dim, rel_types2id_size, focus, pre='rtypes'):
    """Discourse relation types model as Keras Graph."""

    # prepare focus dimensionality
    model.add_node(RepeatVector(rel_types2id_size), name=pre + '_focus_rep', input=focus)
    model.add_node(Permute((2, 1)), name=pre + '_focus', input=pre + '_focus_rep')

    # discourse relation types dense neural network (sample, time_pad, rel_types2id)
    model.add_node(TimeDistributedDense(rel_types2id_size, init='he_uniform'), name=pre + '_dense', input=ins[0])
    model.add_node(Activation('softmax'), name=pre + '_softmax', input=pre + '_dense')

    # multiplication to focus the activations (doc, time_pad, rel_types2id)
    model.add_node(Activation('linear'), name=pre + '_out', inputs=[pre + '_focus', pre + '_softmax'], merge_mode='mul')
    return pre + '_out'

def rel_types_one_model(model, ins, max_len, embedding_dim, rel_types2id_size, focus, pre='rtypesone'):
    """Discourse relation types model to return one output as Keras Graph."""

    # forward recurrent layer returning one last output (sample, rel_types2id)
    model.add_node(GRU(rel_types2id_size, return_sequences=False, activation='sigmoid', inner_activation='sigmoid', init='he_uniform', inner_init='orthogonal'), name=pre + '_fwd', input=ins[0])

    # backward recurrent layer returning one last output (sample, rel_types2id)
    model.add_node(GRU(rel_types2id_size, return_sequences=False, activation='sigmoid', inner_activation='sigmoid', init='he_uniform', inner_init='orthogonal', go_backwards=True), name=pre + '_bck', input=ins[0])

    # join activations (sample, rel_types2id)
    model.add_node(Activation('linear'), name=pre + '_out', inputs=[pre + '_fwd', pre + '_bck'], merge_mode='ave')
    return pre + '_out'

### Build index

def build_rel_types2id(rel_types, max_size=None, min_count=1, rel_types2id=None):
    """Build vocabulary index for all discourse relation types (reserved ids: 0 = missing, 1 = out-of-vocabulary)."""

    return build_index(rel_types, max_size=max_size, min_count=min_count, index=rel_types2id)

### Encode data

def encode_x_rel_types(word_metas_slice, rel_types2id, rel_types2id_size, max_len, filter_prefixes=None, oov_key=""):
    """Encode discourse relation types as normalized vectors (sample, time_pad, rel_types2id)."""

    # crop sequence if needed
    word_metas_slice = word_metas_slice[:max_len]

    # encode all relation types in same row
    x = np.zeros((max_len, rel_types2id_size), dtype=np.float32)

    for i in range(max_len):
        if i < len(word_metas_slice):
            # mark all relation types
            tags = filter_tags(word_metas_slice[i]['RelationTags'], filter_prefixes)
            for rel_tag in tags:
                rel_type, _, _, _ = tag_to_rtsip(rel_tag)

                try:
                    x[i, rel_types2id[rel_type]] += 1.
                except KeyError:  # missing in vocabulary
                    x[i, rel_types2id[oov_key]] += 1.

        if i >= len(word_metas_slice) or not filter_tags(word_metas_slice[i]['RelationTags'], filter_prefixes):
            # no relation types present
            x[i, rel_types2id[None]] += 1.

    # normalize by rows to [0,1] interval
    x_sum = np.sum(x, axis=1)
    x2 = (x.T / x_sum).T
    x2[x_sum == 0.] = x[x_sum == 0.]  # prevent NaN
    return x2

def decode_x_rel_types(x_rel_types, token_range, relation, rel_types2id, rel_types2id_size):
    """Decode one discourse relation type for a given relation spans."""

    # sum type predictions for relation tokens
    totals = np.zeros((rel_types2id_size,))
    for i, token_id in enumerate(token_range):
        if token_id in relation['Arg1'] or token_id in relation['Arg2'] or token_id in relation['Connective'] or token_id in relation['Punctuation']:
            totals += x_rel_types[i]
            #XXX: / np.max(x_rel_types[i])

    # return most probable type
    rel_type = None
    max_total = -1.
    for t, j in rel_types2id.items():
        if totals[j] > max_total:
            max_total = totals[j]
            rel_type = t
    return rel_type, totals

def encode_x_rel_types_one(rel_type, rel_types2id, rel_types2id_size, oov_key=""):
    """Encode discourse relation types as one normalized vector (sample, rel_types2id)."""

    # one-hot encode type
    try:
        i = rel_types2id[rel_type]
    except KeyError:  # missing in vocabulary
        i = rel_types2id[oov_key]
    x = np.zeros((rel_types2id_size,), dtype=np.float32)
    x[i] = 1.
    return x

def decode_x_rel_types_one(x_rel_types_one, rel_types2id, rel_types2id_size):
    """Decode one discourse relation type from a normalized vector."""

    # normalize by rows to [0,1] interval
    x_sum = np.sum(x_rel_types_one)
    totals = x_rel_types_one / x_sum
    totals[x_sum == 0.] = x_rel_types_one[x_sum == 0.]  # prevent NaN

    # return most probable type
    rel_type = None
    max_total = -1.
    for t, j in rel_types2id.items():
        if totals[j] > max_total:
            max_total = totals[j]
            rel_type = t
    return rel_type, totals

### Tests

def test_build_rel_types2id():
    rel_types = {14903: 'Implicit', 14904: 'Explicit', 14878: 'Explicit'}
    t_rel_types2id = {None: 0, "": 1, "Explicit": 2, "Implicit": 3}
    t_rel_types2id_size = len(t_rel_types2id)

    rel_types2id, rel_types2id_size = build_rel_types2id(rel_types)
    assert rel_types2id == t_rel_types2id
    assert rel_types2id_size == t_rel_types2id_size

def test_encode_x_rel_types():
    word_metas_slice = [
        {'RelationIDs': [14903], 'SentenceID': 30, 'RelationTags': ['Implicit:Comparison.Contrast:14903:Arg1'], 'DocID': 'wsj_1000', 'TokenID': 854, 'SentenceOffset': 827, 'Text': 'trading', 'ParagraphID': 13, 'RelationSpans': ['Arg1']},
        {'RelationIDs': [], 'SentenceID': 30, 'RelationTags': [], 'DocID': 'wsj_1000', 'TokenID': 855, 'SentenceOffset': 827, 'Text': '.', 'ParagraphID': 13, 'RelationSpans': []},
        {'RelationIDs': [14903, 14904], 'SentenceID': 31, 'RelationTags': ['Implicit:Comparison.Contrast:14903:Arg2', 'Explicit:Comparison.Concession:14904:Arg1'], 'DocID': 'wsj_1000', 'TokenID': 859, 'SentenceOffset': 857, 'Text': 'that', 'ParagraphID': 13, 'RelationParts': ['Arg2', 'Arg1']},
        {'RelationIDs': [14903, 14904], 'SentenceID': 31, 'RelationTags': ['Implicit:Comparison.Contrast:14903:Arg2', 'Explicit:Comparison.Concession:14904:Arg1'], 'DocID': 'wsj_1000', 'TokenID': 860, 'SentenceOffset': 857, 'Text': '``', 'ParagraphID': 13, 'RelationSpans': ['Arg2', 'Arg1']},
    ]
    rel_types2id = {None: 0, '': 1, 'Implicit': 2, 'Explicit': 3, 'EntRel': 4}
    rel_types2id_size = len(rel_types2id)
    max_len_0 = 3
    max_len_1 = 5
    t_x_0 = [
        [0, 0, 1, 0, 0],
        [1, 0, 0, 0, 0],
        [0, 0, 0.5, 0.5, 0],
    ]
    t_x_1 = [
        [0, 0, 1, 0, 0],
        [1, 0, 0, 0, 0],
        [0, 0, 0.5, 0.5, 0],
        [0, 0, 0.5, 0.5, 0],
        [1, 0, 0, 0, 0],
    ]

    x_0 = encode_x_rel_types(word_metas_slice, rel_types2id, rel_types2id_size, max_len_0)
    assert (x_0 == t_x_0).all()

    x_1 = encode_x_rel_types(word_metas_slice, rel_types2id, rel_types2id_size, max_len_1)
    assert (x_1 == t_x_1).all()

def test_decode_x_rel_types():
    rel_types2id = {None: 0, '': 1, 'Implicit': 2, 'Explicit': 3, 'EntRel': 4}
    rel_types2id_size = len(rel_types2id)
    relation = {
        'Arg1': [854],
        'Arg1Len': 1,
        'Arg2': [859, 860],
        'Arg2Len': 2,
        'Connective': [],
        'ConnectiveLen': 0,
        'Punctuation': [],
        'PunctuationLen': 0,
        'PunctuationType': '',
        'DocID': 'wsj_1000',
        'ID': 14903,
        'TokenMin': 854,
        'TokenMax': 861,
        'TokenCount': 3,
    }
    token_start = 854
    token_end = 861
    x_rel_types_0 = [
        [0., 0., 1., 0., 0.],
        [1., 0., 0., 0., 0.],
        [1., 0., 0., 0., 0.],
        [1., 0., 0., 0., 0.],
        [1., 0., 0., 0., 0.],
        [0., 0., 0.5, 0.5, 0.],
        [0., 0., 0.5, 0.5, 0.],
    ]
    t_type_0 = 'Implicit'
    x_rel_types_1 = [
        [0., 0.25, 0., 0.75, 0.],
        [0., 0., 1., 0., 0.],
        [0., 0., 0., 0., 1.],
        [0., 0., 0., 1., 0.],
        [1., 0., 0., 0., 0.],
        [0., 0.5, 0., 0.5, 0.],
        [0., 0., 0., 0.5, 0.5],
    ]
    t_type_1 = 'Explicit'

    type_0, totals_0 = decode_x_rel_types(x_rel_types_0, range(token_start, token_end), relation, rel_types2id, rel_types2id_size)
    assert type_0 == t_type_0

    type_1, totals_1 = decode_x_rel_types(x_rel_types_1, range(token_start, token_end), relation, rel_types2id, rel_types2id_size)
    assert type_1 == t_type_1

def test_encode_x_rel_types_one():
    rel_types2id = {None: 0, '': 1, 'Implicit': 2, 'Explicit': 3, 'EntRel': 4}
    rel_types2id_size = len(rel_types2id)
    rel_type_0 = "Implicit"
    t_x_0 = [0, 0, 1, 0, 0]
    rel_type_1 = "AltLex"
    t_x_1 = [0, 1, 0, 0, 0]

    x_0 = encode_x_rel_types_one(rel_type_0, rel_types2id, rel_types2id_size)
    assert (x_0 == t_x_0).all()

    x_1 = encode_x_rel_types_one(rel_type_1, rel_types2id, rel_types2id_size)
    assert (x_1 == t_x_1).all()

def test_decode_x_rel_types_one():
    rel_types2id = {None: 0, '': 1, 'Implicit': 2, 'Explicit': 3, 'EntRel': 4}
    rel_types2id_size = len(rel_types2id)
    x_rel_types_one_0 = [0, 0, 1, 0, 0]
    t_type_0 = "Implicit"
    x_rel_types_one_1 = [0, 1, 0, 0, 0]
    t_type_1 = ""

    type_0, totals_0 = decode_x_rel_types_one(x_rel_types_one_0, rel_types2id, rel_types2id_size)
    assert type_0 == t_type_0

    type_1, totals_1 = decode_x_rel_types_one(x_rel_types_one_1, rel_types2id, rel_types2id_size)
    assert type_1 == t_type_1

if __name__ == '__main__':
    import pytest
    pytest.main(['-s', __file__])


import requests
import pickle
url = "https://bj.lianjia.com/zufang/"
response = requests.get(url)
print(response)
print(response.encoding)
html = response.text

with open('html.pkl', 'wb') as handle:
    pickle.dump(html, handle)


import matplotlib.pyplot as plt

# salario = np.array([1100, 1500, 1300])
# print(salario)
# plt.plot( salario,c='r', ls= '--', marker='s' )
# plt.show()

def gerarGrafico( dataList ):

    print( dataList )

    plt.plot( dataList ,c='r', ls= '--', marker='s' )
    plt.show()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('archives', '0011_archivedphoneuser_additional_due_date'),
    ]

    operations = [
        migrations.AddField(
            model_name='archivedphoneuser',
            name='status',
            field=models.IntegerField(default=0, verbose_name=b'stato', choices=[(0, b'nuovo arrivo'), (1, b'in attesa di giudizio'), (2, b'definitivo')]),
        ),
    ]


from time import sleep
from typing import Set, Tuple, Union
from dotenv import load_dotenv
import os
import tweepy
import re
from tqdm import tqdm

class CurrentUser:
    def __init__(
        self, client: tweepy.Client, api: tweepy.API, id: str
    ) -> None:
        self.client: tweepy.Client = client
        self.api: tweepy.API = api
        self.id: str = id

class PyTweeBlockError(Exception):
    pass

def main() -> None:
    curr_user = generate_current_user()
    keep_going = True
    while keep_going:
        keep_going = main_menu(curr_user)

def main_menu(curr: CurrentUser) -> bool:
    print("Please enter with the @ you want to block, or a tweet link.")
    user_input = input().strip()
    if re.match(r"https:\/\/twitter.com\/\w+\/status\/\w+", user_input):
        blocklist = generate_tweet_blocklist(curr, user_input)
        block_users(curr, blocklist)
    elif re.match(r"(@\w+)|(\w+)", user_input):
        blocklist = generate_account_blocklist(curr, user_input)
        block_users(curr, blocklist)
    answer = ""
    while answer not in ("y", "n"):
        answer = (
            input("Do you want to mass block another account? (y\\n)")
            .strip()
            .lower()
        )
    return True if answer == "y" else False

def block_users(curr: CurrentUser, blocklist: Set[str]) -> None:
    curr.client.wait_on_rate_limit = False
    use_api_2 = True
    for id in tqdm(blocklist):
        if use_api_2:
            try:
                curr.client.block(id)
            except tweepy.errors.TooManyRequests:
                curr.api.create_block(user_id=id)
                use_api_2 = False
        else:
            curr.api.create_block(user_id=id)

def generate_account_blocklist(curr: CurrentUser, account: str) -> Set[str]:
    account = account.removeprefix("@")
    account: tweepy.User = curr.client.get_user(
        username=account, user_auth=True
    ).data
    users_following = collect_set_paginator(
        curr.client.get_users_followers,
        account.id,
        max_results=1000,
    )
    users_to_not_block = collect_set_paginator(
        curr.client.get_users_followers,
        curr.id,
        user_auth=True,
        max_results=1000,
    ) | collect_set_paginator(
        curr.client.get_users_following,
        curr.id,
        user_auth=True,
        max_results=1000,
    )
    block_list = (users_following | set([account.id])) - users_to_not_block
    return block_list

def generate_tweet_blocklist(curr: CurrentUser, tweet_link: str) -> Set[str]:
    account = ""
    tweet_id = ""
    try:
        account = re.findall(r"(?<=twitter\.com\/)\w+", tweet_link)[0]
        tweet_id = re.findall(r"(?<=status\/)\d+", tweet_link)[0]
    except IndexError:
        raise PyTweeBlockError(
            "Error extracting account name and tweet id from link"
        )
    account: tweepy.User = curr.client.get_user(
        username=account, user_auth=True
    ).data
    liking_users = collect_set(
        curr.client.get_liking_users,
        tweet_id,
    )
    retweeting_users = collect_set(
        curr.client.get_retweeters,
        tweet_id,
    )
    users_following = collect_set_paginator(
        curr.client.get_users_followers,
        account.id,
        user_auth=True,
        max_results=1000,
    )
    users_to_not_block = collect_set_paginator(
        curr.client.get_users_followers,
        curr.id,
        user_auth=True,
        max_results=1000,
    ) | collect_set_paginator(
        curr.client.get_users_following,
        curr.id,
        user_auth=True,
        max_results=1000,
    )
    block_list = (
        liking_users | users_following | retweeting_users | set([account.id])
    ) - users_to_not_block
    return block_list

def collect_set(method, *args, **kwargs) -> Set[str]:
    return set(map(lambda user: user.id, method(*args, **kwargs).data))

def collect_set_paginator(method, *args, **kwargs) -> Set[str]:
    to_return = set()
    for response in tweepy.Paginator(method, *args, **kwargs):
        to_return |= set(
            map(
                lambda user: user.id,
                response.data,
            )
        )
    #        print("Able to get Paginator once!")
    return to_return

def get_pin_input(url: str) -> str:
    """
    Basic while loop to get the pin from the user
    Returns the identifier stripped and only if it's only digits
    """
    while True:
        print(url)
        verifier = input("Please enter the PIN code: ")
        verifier = verifier.strip()
        if verifier.isdigit():
            return verifier
        else:
            print("Please input only the PIN code")

def generate_current_user() -> CurrentUser:
    load_dotenv()
    consumer_key = os.getenv("TWITTER_API_KEY")
    consumer_secret = os.getenv("TWITTER_API_SECRET")
    bearer_token = os.getenv("TWITTER_BEARER_TOKEN")
    auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
    verifier = get_pin_input(auth.get_authorization_url())
    user_key, user_secret = auth.get_access_token(verifier)
    client = tweepy.Client(
        consumer_key=consumer_key,
        consumer_secret=consumer_secret,
        access_token=user_key,
        access_token_secret=user_secret,
        wait_on_rate_limit=True,
        bearer_token=bearer_token,
    )
    api = tweepy.API(auth)
    user_id = api.verify_credentials().id_str
    print("Login successful!")
    return CurrentUser(client, api, user_id)

if __name__ == "__main__":
    main()


#!/usr/bin/env python3

import sys
import uuid

if __name__ == "__main__":
    if (len(sys.argv) < 2):
        print(f"Usage: {sys.argv[0]} <UUID>")
        sys.exit(1)
    uu = uuid.UUID(sys.argv[1])
    byte_string = "".join(f"\\x{b:02x}" for b in uu.bytes)
    print("#define UUID_" + str(uu).replace("-", "_") + f" (const unsigned char *) \"{byte_string}\"")


import logging
import numpy as np
from mlagents.envs import UnityEnvironment

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("RandomMaze")

class RandomMazeEnv(object):

    def __init__(self, env_file_name):

        self._env = UnityEnvironment(env_file_name)
        self._brain_name = self._env.external_brain_names[0]
        brain = self._env.brains[self._brain_name]

        height = brain.camera_resolutions[0]['height']
        width = brain.camera_resolutions[0]['width']
        action_space = brain.vector_action_space_size[0]

        self._observation_space = (height, width, 3)
        self._action_space = (action_space,)

        self._seed = None

    def reset(self):

        if self._seed is None:
            rseed = np.random.randint(1e6)
        else:
            rseed = self._seed
        assert isinstance(rseed, int), "random seed must be a integer"
        reset_params = dict(RandomSeed = rseed)
        self._env.reset(config = reset_params)[self._brain_name]
        ob, reward, done, info = self.step(np.zeros(self._action_space))
        return ob, reward, done, info

    def step(self, action):

        assert isinstance(action, np.ndarray), "action must be a numpy.ndarry"
        assert action.shape == self._action_space, "action.shape doesn't match env.action_space"

        infos = self._env.step(action)[self._brain_name]
        ob, reward, done, info = self.extract_infos(infos)

        return ob, reward, done, info

    def extract_infos(self, infos):

        ob = infos.visual_observations[0].squeeze()
        reward = infos.rewards[0]
        done = infos.local_done[0]
        info = {}

        return ob, reward, done, info

    def seed(self, seed=None):

        assert isinstance(seed, int), "random seed must be a integer"
        self._seed = seed
        logger.warn("New seed %d will be used after next reset." % self._seed)

    def close(self):
        self._env.close()

    def sample_action(self):
        return np.random.uniform(size=self._action_space)

    @property
    def observation_space(self):
        return self._observation_space

    @property
    def action_space(self):
        return self._action_space