"""
    Created by howie.hu at 2022-05-12.
    Description: 基于Ruia爬虫框架的微信公众号爬虫
        原始数据来源于：https://mp.data258.com/mp/search?type=category&key=老胡的储物柜&sort=
        ! 注意，该目标网站有比较严厉的反爬措施，单IP只能访问15次，此功能需要代理池
        - 运行: 根目录执行，其中环境文件pro.env根据实际情况选择即可
        - 命令: PIPENV_DOTENV_LOCATION=./dev.env pipenv run python src/collector/wechat/data258_ruia_start.py
        - 结果示例：
        {
            "doc_date": "2022-01-09 21:20:00",
            "doc_image": "wx_fmt=jpeg",
            "doc_name": "我的周刊（第021期）",
            "doc_ts": 1641734400,
            "doc_link": "",
            "doc_source_meta_list": [
                "howie_locker",
                "编程、兴趣、生活"
            ],
            "doc_des": "奇文共欣赏，疑义相与析",
            "doc_core_html": "hello world",
            "doc_type": "article",
            "doc_author": "howie6879",
            "doc_source_name": "老胡的储物柜",
            "doc_id": "3b6b3dd93b58164f0f60403b06ef689a",
            "doc_source": "liuli_wechat",
            "doc_source_account_nick": "howie_locker",
            "doc_source_account_intro": "编程、兴趣、生活",
            "doc_content": "hello world",
            "doc_keywords": ""
        }
    Changelog: all notable changes to this file will be documented
"""
import asyncio
import re

from urllib.parse import urljoin

import execjs

from ruia import Response, Spider
from ruia_ua import middleware as ua_middleware

from src.collector.utils import load_data_to_articlles
from src.collector.wechat.items import (
    Data258WechatItem,
    Data258WechatListItem,
    WechatItem,
)
from src.processor import html_to_text_h2t
from src.utils.log import LOGGER
from src.utils.tools import md5_encryption

def exec_js_data258(html: str) -> str:
    """返回js执行结果"""
    t_list = re.findall(r"setTimeout\(function\(\){(.*?);},", html)
    mp_res_list = []
    for i in t_list:
        js_text = """
        window = {};
        location = {
        href: null,
        };
        """
        js_text += re.compile(r"\}\);(.*?)</script>", re.S).search(html)[1]
        js_text += i
        js = execjs.compile(js_text)
        res = js.eval("location")["href"]
        if "www.data258.com" in res:
            break
        else:
            mp_res_list.append(res)

    return mp_res_list[-1] if mp_res_list else None

class Data258WechatSpider(Spider):
    """微信文章爬虫"""

    name = "Data258WechatSpider"
    request_config = {"RETRIES": 3, "DELAY": 3, "TIMEOUT": 10}
    concurrency = 1
    base_url = "https://mp.data258.com/"
    wechat_name = ""
    # aiohttp config
    aiohttp_kwargs = {}

    async def parse(self, response: Response):
        """解析公众号原始链接数据"""
        html = await response.text()
        async for item in Data258WechatItem.get_items(html=html):
            if item.wechat_name == self.wechat_name:
                url = urljoin(self.base_url, item.wehcat_href)
                yield self.request(
                    url=url,
                    callback=self.parse_wechat_articles,
                )
                break

    async def parse_wechat_articles(self, response: Response):
        """解析公众号详情页面，提取历史文章"""
        html = await response.text()

        async for item in Data258WechatListItem.get_items(html=html):
            url = urljoin(self.base_url, item.w_article_href)
            yield self.request(
                url=url,
                headers={"Host": "mp.data258.com", "Referer": response.url},
                callback=self.parse_wechat_url,
                metadata={"wechat_title": item.w_article_title},
            )

    async def parse_wechat_url(self, response: Response):
        """解析公众号文章原始链接"""
        html = await response.text()
        if "今日浏览次数已达上限" in html:
            LOGGER.error(f"微信真实链接提取失败(IP 限制)：{response.metadata['wechat_title']} ")
        else:
            real_wechat_url = await asyncio.coroutine(exec_js_data258)(html)
            if real_wechat_url:
                yield self.request(
                    url=real_wechat_url,
                    callback=self.parse_wechat,
                )
            else:
                LOGGER.error(f"微信真实链接提取失败(js解密)：{response.metadata['wechat_title']} ")

    async def parse_wechat(self, response: Response):
        """解析公众号元数据"""
        html = await response.text()
        wechat_item: WechatItem = await WechatItem.get_item(html=html)
        wechat_data = {
            **wechat_item.results,
            **{
                "doc_id": md5_encryption(f"{wechat_item.doc_name}_{self.wechat_name}"),
                "doc_keywords": "",
                "doc_source_name": self.wechat_name,
                "doc_link": response.url,
                "doc_source": wechat_item.doc_source,
                "doc_source_account_nick": wechat_item.doc_source_account_nick,
                "doc_source_account_intro": wechat_item.doc_source_account_intro,
                "doc_content": html_to_text_h2t(html),
                "doc_html": "",
            },
        }
        await asyncio.coroutine(load_data_to_articlles)(input_data=wechat_data)

def run(collect_config: dict):
    """微信公众号文章抓取爬虫

    Args:
        collect_config (dict, optional): 采集器配置
    """
    s_nums = 0
    wechat_list = collect_config["wechat_list"]
    delta_time = collect_config.get("delta_time", 3)
    for wechat_name in wechat_list:
        Data258WechatSpider.wechat_name = wechat_name
        Data258WechatSpider.request_config = {
            "RETRIES": 3,
            "DELAY": delta_time,
            "TIMEOUT": 5,
        }
        t_url = (
            f"https://mp.data258.com/mp/search?type=category&key={wechat_name}&sort="
        )
        Data258WechatSpider.start_urls = [t_url]
        try:
            Data258WechatSpider.start(middleware=ua_middleware)
            s_nums += 1
        except Exception as e:
            err_msg = f"😿 公众号->{wechat_name} 文章更新失败! 错误信息: {e}"
            LOGGER.error(err_msg)

    msg = f"🤗 微信公众号文章更新完毕({s_nums}/{len(wechat_list)})!"
    LOGGER.info(msg)

if __name__ == "__main__":
    t_collect_config = {"wechat_list": ["老胡的储物柜"], "delta_time": 5}
    run(t_collect_config)
    # wechat_name = "老胡的储物柜"
    # t_url = f"https://mp.data258.com/mp/search?type=category&key={wechat_name}&sort="
    # Data258WechatSpider.start_urls = [t_url]
    # Data258WechatSpider.wechat_name = wechat_name
    # Data258WechatSpider.start(middleware=ua_middleware)


from rest_framework import status, mixins
from rest_framework.response import Response
from rest_framework.views import APIView
from rest_framework.viewsets import GenericViewSet

from devices.models import Device
from devices.serializers import DeviceSerializer
from .authentication import CsrfExemptSessionAuthentication, SiriAuthentication, WebhookAuthentication
from .serializers import SiriSerializer, WebhookSerializer
from .utils import update_device_status

class SiriAPIView(APIView):
    authentication_classes = (CsrfExemptSessionAuthentication, SiriAuthentication)

    def post(self, request):
        serializer = SiriSerializer(data=request.data)
        if serializer.is_valid():
            data = serializer.validated_data
            device_name = data.get("device_name")
            if device_name:
                device = Device.objects.filter(name=device_name, active=True).first()
                if device:
                    update_device_status(device.unique_id, data["state"])
                    return Response(serializer.data, status=status.HTTP_200_OK)

            else:
                devices = Device.objects.filter(active=True)
                if devices:
                    for device in devices:
                        update_device_status(device.unique_id, data["state"])

                    return Response(serializer.data, status=status.HTTP_200_OK)

        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

    def get_queryset(self):
        return Device.objects.all()

class WebhookAPIView(APIView):
    authentication_classes = (CsrfExemptSessionAuthentication, WebhookAuthentication)

    def post(self, request):
        serializer = WebhookSerializer(data=request.data)
        if serializer.is_valid():
            data = serializer.validated_data
            device = Device.objects.filter(unique_id=data["unique_id"]).first()
            if device:
                active = data.get("active")
                state = data.get("state")
                auto = data.get("auto")
                if active is not None:
                    device.active = active
                if state is not None:
                    device.state = state
                if auto is not None:
                    device.auto = auto
                device.save()
                return Response(serializer.data, status=status.HTTP_200_OK)

        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

    def get_queryset(self):
        return Device.objects.all()

class ListDevicesViewSet(
    mixins.ListModelMixin,
    GenericViewSet,
):
    authentication_classes = (CsrfExemptSessionAuthentication, WebhookAuthentication)
    serializer_class = DeviceSerializer
    queryset = Device.objects.all()

    def get_queryset(self):
        return Device.objects.filter(active=True)


from enum import Enum
from abc import ABC, abstractmethod

from uncertainty.utypes import *

class IncubatorState(Enum):
    Heating = 1
    CoolingDown = 2

class ActiveObject:

    @abstractmethod
    def action(self):
        pass

class Clock:

    def __init__(self):
        self.now : float = 0.0
        self.timeStep : float = 3
        self.activeObjects : list[ActiveObject] = []

    def tick(self):
        self.now = self.now + self.timeStep
        for o in filter(lambda p : not isinstance(p, Controller), self.activeObjects):
            o.action()

        for o in filter(lambda p : isinstance(p, Controller), self.activeObjects):
            o.action()

    def run(self, n:int):
        for i in range(n):
            self.tick()

    def addObject(self, activeObject):
        if isinstance(activeObject, ActiveObject):
            self.activeObjects.append(activeObject)

class Incubator(ActiveObject):
    def __init__(self):
        super().__init__()
        self.cAir : ufloat #  mass of object inside box * capacity of air inside box
        self.gBox : ufloat #  link the difference between the room temperature and the air temperature inside the box with the energy flowing to the box
        self.voltage : ufloat = ufloat(0, 0.05)
        self.onVoltage : ufloat = ufloat(18, 0.05)
        self.current : ufloat = ufloat(4, 0.05)
        self.tBair : ufloat = ufloat(22.8125, 0.05) #  temperature inside box
        self.tRoom : ufloat = ufloat(20, 0.05) #  temperature of the room
        self.currentState : IncubatorState = IncubatorState.CoolingDown
        self.numericalPrecision : ufloat = ufloat(0, 0.05)

    @abstractmethod
    def calculateTemperature(self, timeIncrement : ufloat) -> ufloat:
        pass

class Incubator2P(Incubator):
    def init(self):
        super().__init__()
        self.cAir = ufloat(616.56, 0.05)
        self.gBox = ufloat(0.65, 0.05)

    def calculateTemperature(self, timeIncrement : ufloat) -> ufloat:
            (1/self.cAir)*(self.voltage * self.current * timeIncrement - self.gBox * (self.tBair - self.tRoom))

    def action(self):
            self.tBair : self.tBair + self.clock.timeStep*self.calculateTemperature(self.clock.timeStep)

class Incubator4P(Incubator):
    def __init__(self, clock):
        super().__init__()
        self.tHeater : ufloat = ufloat(25,0.05)
        self.cAir: ufloat = ufloat(486.12, 0.05)
        self.gBox: ufloat = ufloat(0.856, 0.05)
        self.cHeater: ufloat = ufloat(33.65, 0.05)
        self.gHeater: ufloat = ufloat(0.87, 0.05)
        self.clock: Clock = clock

    def caculateHeaterTemperature(self, timeIncrement : ufloat) -> ufloat:
        return (1/self.cHeater) * (self.voltage * self.current * timeIncrement - self.gHeater * (self.tHeater - self.tBair))

    def calculateTemperature(self, timeIncrement : ufloat) -> ufloat:
        return (1/self.cAir) * (self.gHeater * (self.tHeater-self.tBair) - self.gBox * (self.tBair - self.tRoom))

    def action(self):
        self.tHeater = self.tHeater + self.clock.timeStep * self.caculateHeaterTemperature(self.clock.timeStep)
        self.tBair = self.tBair + self.clock.timeStep * self.calculateTemperature(self.clock.timeStep)

class Controller(ActiveObject):
    def __init__(self, clock, incubator):
        super().__init__()
        self.desiredTemp : ufloat = ufloat(28, 0.05)
        self.lowerBound : ufloat = ufloat(3, 0.05)
        self.tolerance : float = 0.99999
        self.incubator: Incubator = incubator
        self.clock: Clock = clock

    def action(self):
        if self.incubator.currentState == IncubatorState.Heating and (self.incubator.tBair >= self.desiredTemp).confidence > self.tolerance:
            self.incubator.currentState = IncubatorState.CoolingDown
            self.incubator.voltage = ufloat(0, 0.05)
        elif (self.incubator.currentState == IncubatorState.CoolingDown) and ((self.incubator.tBair <= (self.desiredTemp - self.lowerBound)).confidence > self.tolerance):
            self.incubator.currentState = IncubatorState.Heating
            self.incubator.voltage = self.incubator.onVoltage

        print(str(self.clock.now) + '\t| ' + str(self.incubator.tBair.value) + '\t| ' + str(self.incubator.tBair.uncertainty))

if __name__ == "__main__":
    clock = Clock()
    incubator4p = Incubator4P(clock)
    controller = Controller(clock, incubator4p)

    clock.addObject(incubator4p)
    clock.addObject(controller)

    print("Time\t| Temperature\t\t| Uncertainty")
    print("------------------------------------------------------")
    clock.run(15)


# ch7/example1.py

import multiprocessing

class ReductionConsumer(multiprocessing.Process):

    def __init__(self, task_queue, result_queue):
        multiprocessing.Process.__init__(self)
        self.task_queue = task_queue
        self.result_queue = result_queue

    def run(self):
        pname = self.name
        print('Using process %s...' % pname)

        while True:
            num1 = self.task_queue.get()
            if num1 is None:
                print('Exiting process %s.' % pname)
                self.task_queue.task_done()
                break

            self.task_queue.task_done()
            num2 = self.task_queue.get()
            if num2 is None:
                print('Reaching the end with process %s and number %i.' % (pname, num1))
                self.task_queue.task_done()
                self.result_queue.put(num1)
                break

            print('Running process %s on numbers %i and %i.' % (pname, num1, num2))
            self.task_queue.task_done()
            self.result_queue.put(num1 + num2)

def reduce_sum(array):
    tasks = multiprocessing.JoinableQueue()
    results = multiprocessing.JoinableQueue()
    result_size = len(array)

    n_consumers = multiprocessing.cpu_count()

    for item in array:
        results.put(item)

    while result_size > 1:
        tasks = results
        results = multiprocessing.JoinableQueue()

        consumers = [ReductionConsumer(tasks, results) for i in range(n_consumers)]
        for consumer in consumers:
            consumer.start()

        for i in range(n_consumers):
            tasks.put(None)

        tasks.join()
        result_size = result_size // 2 + (result_size % 2)
        #print('-' * 40)

    return results.get()

my_array = [i for i in range(20)]

result = reduce_sum(my_array)
print('Final result: %i.' % result)


import json
import time
from socket import *
from ipaddress import ip_address
import re, argparse, threading
from ipaddress import ip_address
from netaddr import IPNetwork
import requests
# ***************************************
bufferSize  = 1024
serverPort = 67
server_ip = "127.0.0.1"
msgFromServer = "Hello UDP Client"
bytesToSend = str.encode(msgFromServer)
reservation_list = []
banned_list = []
gateway = "192.168.1.1"
dns = ["9.7.10.15",	"9.7.10.16","9.7.10.18"]
# ***************************************
IP_list = {}
# start_time = 0
running = True

def packet_analyser(packet):
    OP = packet[0]
    HTYPE = packet[1]
    HLEN = packet[2]
    HOPS = packet[3]
    XID = packet[4:8]
    SECS = packet[8:10]
    FLAGS = packet[10:12]
    CIADDR = packet[12:16]
    YIADDR = packet[16:20]
    SIADDR = packet[20:24]
    GIADDR = packet[24:28]
    CHADDR = packet[28:28 + 16 + 192]
    magic_cookie = packet[236:240]
    DHCPoptions = packet[240:]

    return OP, HTYPE, HLEN, HOPS, XID, SECS, FLAGS, CIADDR, YIADDR, SIADDR, GIADDR, CHADDR, magic_cookie, DHCPoptions

def DHCP_offer(xid, ciaddr, chaddr, magic_cookie, ip):
	OP = bytes([0x02])
	HTYPE = bytes([0x01])
	HLEN = bytes([0x06])
	HOPS = bytes([0x00])
	XID = xid
	SECS = bytes([0x00, 0x00])
	FLAGS = bytes([0x00, 0x00])
	CIADDR = ciaddr
	YIADDR = inet_aton(ip) 													#adresse a donner
	SIADDR = inet_aton(server_ip)
	GIADDR = bytes([0x00, 0x00, 0x00, 0x00])
	CHADDR = chaddr
	DHCPoptions1 = bytes([53, 1, 2])
	DHCPoptions2 = bytes([1 , 4]) + inet_aton(subnet_mask)				# read from json file
	DHCPoptions3 = bytes([3 , 4 ]) + inet_aton(gateway) 				# gateway/router
	DHCPOptions4 = bytes([51 , 4]) + ((lease_time).to_bytes(4, byteorder='big')) 	#IP address lease time
	DHCPOptions5 = bytes([54 , 4]) + inet_aton(server_ip) 				# DHCP server
	DHCPOptions6 = bytes([6, 4 * len(dns)]) 							#DNS servers
	# for i in dns:
		# DHCPOptions6 += i

	ENDMARK = bytes([0xff])

	package = OP + HTYPE + HLEN + HOPS + XID + SECS + FLAGS + CIADDR + YIADDR + SIADDR + GIADDR + CHADDR + magic_cookie + DHCPoptions1 + DHCPoptions2 + DHCPoptions3 + DHCPOptions4 + DHCPOptions5 + DHCPOptions6 + ENDMARK
	return package

def DHCP_ack(xid, ciaddr, chaddr, magicookie, ip):
	OP = bytes([0x02])
	HTYPE = bytes([0x01])
	HLEN = bytes([0x06])
	HOPS = bytes([0x00])
	XID = xid
	SECS = bytes([0x00, 0x00])
	FLAGS = bytes([0x00, 0x00])
	CIADDR = ciaddr
	YIADDR = inet_aton(ip) 													#adresse a donner
	SIADDR = inet_aton(server_ip)
	GIADDR = bytes([0x00, 0x00, 0x00, 0x00])
	CHADDR = chaddr
	Magiccookie = magicookie
	DHCPoptions1 = bytes([53 , 1 , 5]) 										#DHCP ACK(value = 5)
	DHCPoptions2 = bytes([1 , 4]) + inet_aton(subnet_mask)				# subnet_mask 255.255.255.0
	DHCPoptions3 = bytes([3 , 4 ]) + inet_aton(gateway) 				# gateway
	DHCPoptions4 = bytes([51 , 4]) + ((lease_time).to_bytes(4, byteorder='big'))
	DHCPoptions5 = bytes([54 , 4]) + inet_aton(server_ip) 				# DHCP server
	DHCPOptions6 = bytes([6, 4 * len(dns)]) 							# DNS servers
	ENDMARK = bytes([0xff])

	package = OP + HTYPE + HLEN + HOPS + XID + SECS + FLAGS + CIADDR + YIADDR + SIADDR + GIADDR + CHADDR + Magiccookie + DHCPoptions1 + DHCPoptions2 + DHCPoptions3 + DHCPoptions4 + DHCPoptions5 + DHCPOptions6 + ENDMARK
	return package

def init_IP(mode, Ip_block, Ip_mask):
	global IP_list
	if mode == "range":
		start = ip_address(Ip_block)
		end = ip_address(Ip_mask)
		while start <= end:
			# 1:IP address - 2nd: lease time
			p = str(start)
			IP_list[p] = ("null",0,"null")
			start+=1
	elif mode == "subnet":
		network = IPNetwork('/'.join([ip_block, Ip_mask]))
		generator = network.iter_hosts()
		for ip in list(generator):
			# 1:IP address - 2nd: lease time, 3nd: remaining time
			key = str(ip)
			IP_list[key] = ("null",0,"null")

def get_free_ip():
	global IP_list
	for key, value in IP_list.items() :
		if(value[0] == "null"):
			return key
	return False

def get_ip(mac_address, ip):
	global IP_list
	for key, value in IP_list.items() :
		if(value[0] == mac_address):
			return key

	# if clien't ip request is free, set it.
	if(ip == True):
		if(IP_list.get(ip)[0] == "null"):
			return ip

	return get_free_ip()

def get_banned_adresses():
	return banned_list

def ip_addr_format(address):
    return ('{}.{}.{}.{}'.format(*bytearray(address)))

def mac_addr_format(address):
    address = address.hex()[:16]
    return (':'.join(address[i:i+2] for i in range(0,12,2)))

def update_ip(ip, mac_address, lease, init_time = "null"):
	global IP_list
	IP_list.update({ip: (mac_address,lease, init_time)})
	return

def clientManage(server, address, packet):
	# global start_time
	# print("new Dosscover message")
	dhcp_client_packet = packet_analyser(packet)
	dhcpoptions = packet_analyser(packet)[13]
	dhcpMessageType = dhcpoptions[2]
	dhcpRequestedIp = False
	dest = ('255.255.255.255', address[1])

	# read requested IP address
	for i in range(len(dhcpoptions)):
	    if(dhcpoptions[i:i+2] == bytes([50, 4])):
	        dhcpRequestedIp = ip_addr_format(dhcpoptions[i+2:i+6])

	xid, ciaddr, chaddr, magic_cookie = dhcp_client_packet[4], dhcp_client_packet[7], dhcp_client_packet[11], dhcp_client_packet[12]
	dhcpClientMacAddress = mac_addr_format(chaddr)
	if(dhcpClientMacAddress not in get_banned_adresses()):
		if(dhcpMessageType == 1):
			ip = get_ip(str(dhcpClientMacAddress), dhcpRequestedIp)
			if(ip != False):
				# print("send offer")
				data = DHCP_offer( xid, ciaddr, chaddr, magic_cookie, ip)
				server.sendto(data, dest)
			else:
				print("all ips are occupied")

		if(dhcpMessageType == 3):
			# print("Receive DHCP request.(" + dhcpClientMacAddress + ')')
			ip = get_ip(str(dhcpClientMacAddress), dhcpRequestedIp)
			if(ip != False):
				data = DHCP_ack(xid, ciaddr, chaddr, magic_cookie, ip)
				update_ip(ip, str(dhcpClientMacAddress),lease_time,time.time())
				server.sendto(data, dest)
				# print("send ack message")
				while True:
					current_time = time.time()
					elapsed_time = current_time - IP_list[ip][2]
					if elapsed_time > (IP_list[ip][1]+1):
						update_ip(ip, "null", 0, current_time)
						# print(f"free { ip }")
						break
	else:
		print("this client is banned")
		server.sendto(b"banned", dest)

def get_mac_details(mac_address):
	url = "https://api.macvendors.com/"
	response = requests.get(url + mac_address).text
	return response
	# if response.status_code != 200:
	    # raise Exception("[!] Invalid MAC Address!")
	# return response.content.decode()

def stop():
	global running
	running = False

def get_ip_allocated():
	global IP_list
	package = "IP ADDRESSES  |  MAC ADDRESSES  |  Lease time |  Device Name \n----------------------------- \n"
	for key, value in IP_list.items():
		if(value[0] != "null"):
			package += ("(" + key + ") at " + value[0] + '\n')
			if value[2] != "null":
				package += "remaining time: " + str(time.time() - value[2])+'\n'
			else:
				package+= "reserved \n"
			# try:
			vendor_name = get_mac_details(value[0])
			package += "Device vendor is " + vendor_name +'\n'
			# except:
				# package +=	"Device Name: " + "[!] Invalid MAC Address! \n"
				# package +="****************************\n"
	return package

def get_ip_available():
	global IP_list
	package = "IP availables : "
	for key, value in IP_list.items() :
		if(value[0] == "null"):
			package += ("\t(" + key + ") \n")
	return package

def gui():
	global running
	print("[ stop ]	: stop the DHCP server ")
	print("[ usage ] : show ip assignment ")
	print("[ available ] : show ip still available ")
	print("*******************************************************")
	while running:
		request = input(">>Server: ").lower()
		if(request == "stop"):
			stop()

		elif(request == "usage"):
			print(get_ip_allocated())

		elif(request == "available"):
			print(get_ip_available())
	print("DHCP server gui has stoped")

def dhcp_server():
	global running
	server = socket(AF_INET, SOCK_DGRAM)
	server.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
	server.setsockopt(SOL_SOCKET, SO_BROADCAST, 1)
	server.bind((server_ip, serverPort))
	while running:
		bytesAddressPair = server.recvfrom(bufferSize)
		packet = bytesAddressPair[0]
		address = bytesAddressPair[1]
		newThread = threading.Thread(target=clientManage, args=(server, address, packet))
		newThread.start()
	print("Bye")

if __name__ == '__main__':
	f = open('config.json')
	data = json.load(f)
	pool_mode = data['pool_mode']

	ip_from = data['range']['from']
	ip_to = data['range']['to']

	ip_block = data['subnet']['ip_block']
	subnet_mask = data['subnet']['subnet_mask']

	lease_time = int(data['lease_time'])

	f.close()

	if pool_mode == "range":
		init_IP(pool_mode,ip_from,ip_to)
	else:
		init_IP(pool_mode,ip_block,subnet_mask)

	for pair in data["reservation_list"]:
		p = data["reservation_list"][pair]
		reservation_list.append(p)
		update_ip(p[1],p[0],0)

	for ban in range(len(data["black_list"])):
		mac = data["black_list"][ban]
		banned_list.append(mac)

	server_gui = threading.Thread(target=gui, name='gui')
	server_gui.start()
	dhcp_server()
	server_gui.join()


import shutil
import datetime
import string
import subprocess

import cmn
import glob
import os
import time
from zipfile import ZipFile

log_file = 'zipper.log'

def remove_log():
    global log_file
    with open(log_file, "a+") as f:
        f.write('\n')
        f.write('--------------------------------------------------------------------------------\n')
        f.write('\n')

def log(message):
    global log_file
    strmsg = time.strftime('%Y.%m.%d %T') + ' ' + message + '\n'
    with open(log_file, "a+") as f:
        f.write(strmsg)

def remover(item):
    iterator = 0
    pick = time.time()
    with ZipFile('/u02/app/oracle/oradata/datastore/logs/odadb1_aud/log.zip', 'w') as zip:
        for file in glob.glob(item["file_mask"]):
            if os.stat(file).st_mtime < pick - item["delay_in_days"] * 86400:
                zip.write(file)
                os.remove(file)
                iterator += 1
                if iterator % 1000 == 0:
                    log('temporary deleted files by mask {mask} is {iterator}'
                        .format(mask=item["file_mask"], iterator=iterator))

    log('deleted files by mask {mask} is {iterator}'.format(mask=item["file_mask"], iterator=iterator))

def checkfile(file):
    cmn.log.debug('found %s',file)

def copy_file(source, dest):
    # copy file to destination
    cmn.log.debug('file %s copy to %s' , file, dest)
    if dest[-4]=='.':
        make_dir(generate_mask(dest)[0:string.rfind(generate_mask(dest),'/')])
        shutil.copyfile(source, generate_mask(dest))
    else:
        make_dir(generate_mask(dest))
        shutil.copy(source, generate_mask(dest))

def remove_file(file):
    os.remove(file)

def move_file(source, dest):
    # move file to destination
    cmn.log.debug('file %s move to %s' , source, dest)
    make_dir(generate_mask(dest))
    shutil.move(source, generate_mask(dest))

def find_files_by_date( source, delta):
    cmn.log.debug('find_files_by_date action %s', delta)
    pick = time.time()
    res = []
    if delta is None: delta = 0
    for iter_file in glob.glob(source):
        if os.stat(iter_file).st_mtime < pick - delta * 86400:
            res.append(iter_file)
    return res

            # if action=='check':
            #     checkfile(iter_file)
            # if action == 'cp':
            #     copy_file(iter_file, dest)

def findfiles(action, source, destination,delta):
    cmn.log.debug('findfiles action %s', delta)
    pick = time.time()
    if delta is None: delta = 0
    for iter_file in glob.glob(source):
        if os.stat(iter_file).st_mtime < pick - delta * 86400:
            if action=='check':
                checkfile(iter_file)
            if action == 'cp':
                copy_file(iter_file, destination)

def conf_iterator():
    cmn.log.debug('conf_iterator')
    cmn.log.debug(cmn.config)
    for iterKey, iterValue in cmn.config.items():
       if iterValue['action'] in ('check','cp'):
           findfiles(iterValue['action'],iterValue['source'],iterValue.get('dest') ,iterValue.get('delta'))

def generate_mask(mask):
    cmn.log.debug('generate_mask')
    return generate_date(mask)

def make_dir(dir):
    # create directory if not exists
    if not os.path.exists(dir):
        os.makedirs(dir)

def generate_date(input_string):
    today = datetime.date.today()
    first = today.replace(day=1)
    last_month = first - datetime.timedelta(days=1)
    var_char = last_month.strftime("%Y.%m")
    return input_string.replace("%YM%", var_char)

def copy_to_archive(files, archive):
    make_dir(generate_mask(archive)[0:string.rfind(generate_mask(archive), '/')])
    for file in files:
        with ZipFile(generate_mask(archive), 'a') as zip:
            zip.write(file)
            os.remove(file)

def copy_to_server(file, user, server, path ):
    cmd = 'scp {file} {user}@{server}:{path}'.format(file=file, user=user, server=server, path=path)
    subprocess.call(cmd, shell=True)

def move_to_server(file, user, server, path ):
    copy_to_server(file, user, server, path)
    remove_file(file)

if __name__ == "__main__":
    # cli.read_cli_flags(sys.argv[1:])
    # cli.configlogger(filename=cmn.args['logfile'], level='debug')
    # utl_config.read_config()
    # conf_iterator()
    # make_dir(generate_mask('/u02/app/oracle/oradata/datastore/database_logs/test/%YM%/'))
    # copy_file('/u01/app/oracle/diag/rdbms/eisgs00/eisgs001/trace/alert_eisgs001.log', '/u02/app/oracle/oradata/datastore/database_logs/test/odadb1/%YM%/')
    # move_file('/u02/app/oracle/oradata/datastore/database_logs/test/odadb1/2022.05/alert_eisgs001.log', '/u02/app/oracle/oradata/datastore/database_logs/test/odadb2/%YM%/')
    # copy_to_server('/home/oracle/holdi.sql', 'oracle', 'odadb2','/home/oracle/')
    # remove_file('/home/oracle/holdi.sql')
    # copy_file('/home/oracle/holdi.sql','/home/oracle/holdi1.sql')
    # move_to_server('/home/oracle/holdi1.sql', 'oracle', 'odadb2','/home/oracle/')
    filelist  = find_files_by_date('/u01/app/oracle/admin/eisgs03/adump/*.aud', 1)
    copy_to_archive(filelist, '/u02/app/oracle/oradata/datastore/database_logs/odadb1/%YM%/aud.zip')

    # utl_config.write_config()


"""Fairness metric functions."""
import re
from typing import List, Optional, Tuple

import numpy as np
import pandas as pd
from flwr.common.typing import Metrics, NDArray
from sklearn.metrics import confusion_matrix
from sklearn.neighbors import NearestNeighbors

from inflorescence.dataset import Dataset

def weighted_average(metrics: List[Tuple[int, Metrics]]) -> Metrics:
    """Multiply each metric of each client by number of examples and divide by total sample size.
    Sample size for a metric will only include clients who reported that metric.
    """
    values = {}  # key: metric name, value: weighted sum of metric values across clients
    num_examples = {}  # key: metric name, value: number of examples for that metric
    for n, met in metrics:
        for k, v in met.items():
            values.setdefault(k, 0)
            num_examples.setdefault(k, 0)
            values[k] += n * v
            num_examples[k] += n
    return {k: v / num_examples[k] if num_examples[k] >= 1 else np.nan for k, v in values.items()}

def sum_counts(metrics: List[Tuple[int, Metrics]]):
    """Just sum the metrics, useful for counts."""
    values = {}
    for _, metric in metrics:
        for k, v in metric.items():
            values.setdefault(k, 0)
            values[k] += v
    return values

def aggregate_binary_cm(metrics: List[Tuple[int, Metrics]]) -> Metrics:
    """Sum the confusion matrix counts but weighted average any other metrics."""
    values = {}
    num_examples = {}
    regex = "(tn|tp|fn|fp|cm)"
    regex2 = "(true_negatives|true_positives|false_negatives|false_positives)"
    for n, met in metrics:
        if "cluster" in met:
            # tally up the count of clients per cluster
            cluster_num = met["cluster"]
            key = f"cluster_{cluster_num:02}"
            values[key] = values.get(key, 0) + 1
        for k, v in met.items():
            if not k.startswith("cluster"):  # Don't aggregate cluster number
                values.setdefault(k, 0)
                num_examples.setdefault(k, 0)
                if re.search(regex, k) or re.search(regex2, k):
                    values[k] += v
                else:
                    values[k] += v * n
                num_examples[k] += n
    for k, v in values.items():
        if not re.search(regex, k) and not re.search(regex2, k) and not k.startswith("cluster"):
            values[k] = v / num_examples[k] if num_examples[k] >= 1 else np.nan

    return values

def ppp_cm(tn: int, fp: int, fn: int, tp: int):
    """Calculate probability of positive prediction from a confusion matrix."""
    return (tp + fp) / (tn + fp + fn + tp)

def tpr_cm(tn: int, fp: int, fn: int, tp: int):
    """Calculate true positive rate from a confusion matrix."""
    return tp / (tp + fn)

def fpr_cm(tn: int, fp: int, fn: int, tp: int):
    """Calculate false positive rate from a confusion matrix."""
    return fp / (fp + tn)

def ppv_cm(tn: int, fp: int, fn: int, tp: int):
    """Calculate positive predictive value from a confusion matrix."""
    return tp / (tp + fp)

def true_negatives(yhat: NDArray, y: NDArray):
    return ((1 - y) * (1 - yhat)).sum()

def true_positives(yhat: NDArray, y: NDArray):
    return (y * yhat).sum()

def false_positives(yhat: NDArray, y: NDArray):
    return ((1 - y) * yhat).sum()

def false_negatives(yhat: NDArray, y: NDArray):
    return (y * (1 - yhat)).sum()

def acc_cm(tn: int, fp: int, fn: int, tp: int):
    """Calculate binary classifier accuracy from a confusion matrix."""
    return (tn + tp) / (tn + fp + fn + tp)

def disparate_impact_ratio(yhat: NDArray, y: NDArray, a: NDArray):
    """Disparate impact ratio metric.
    Ratio of probability of positive prediction by group status.
    """
    y_hat = yhat >= 0.5
    groups = np.unique(a)
    num_groups = len(groups)
    if num_groups < 2:
        return {}
    result = np.ones((num_groups))
    for i, group in enumerate(groups):
        ingroup = y_hat[a == group]
        outgroup = y_hat[a != group]
        ingroup_ppp = ingroup.sum() / len(ingroup)
        outgroup_ppp = outgroup.sum() / len(outgroup)
        result[i] = ingroup_ppp / outgroup_ppp
    return {f"disparate_impact_ratio_{i:02}": v for i, v in enumerate(result)}

def equal_odds_ratio(yhat: NDArray, y: NDArray, a: NDArray):
    """
    Lesser of ratios of true positive and false positive rates by protected group status
    """
    groups = np.unique(a)
    num_groups = len(groups)
    if num_groups < 2:
        return {}
    result = np.ones((num_groups))
    for i, group in enumerate(groups):
        tpr_a = true_positive_rate(yhat[a == group], y[a == group])
        fpr_a = false_positive_rate(yhat[a == group], y[a == group])
        tpr_nota = true_positive_rate(yhat[a != group], y[a != group])
        fpr_nota = false_positive_rate(yhat[a != group], y[a != group])
        result[i] = min(np.divide(tpr_a, tpr_nota), np.divide(fpr_a, fpr_nota))
    return {f"equal_odds_ratio_{i:02}": v for i, v in enumerate(result)}

def binary_confusion_matrix(yhat: NDArray, y: NDArray):
    """Calculate the confusion matrix."""
    tp = (y * yhat).sum()
    tn = ((1 - y) * (1 - yhat)).sum()
    fp = ((1 - y) * yhat).sum()
    fn = (y * (1 - yhat)).sum()
    return {"tn": tn, "fp": fp, "fn": fn, "tp": tp}

def group_binary_confusion_matrix(yhat: NDArray, y: NDArray, a: NDArray):
    """Calculate the confusion matrix."""
    groups = np.unique(a)
    result = {}
    for i, group in enumerate(groups):
        group_cm = binary_confusion_matrix(yhat[a == group], y[a == group])
        result.update({f"{k}_group_{i:02}": v for k, v in group_cm.items()})
    return result

def multiclass_confusion_matrix(yhat: NDArray, y: NDArray):
    cm = confusion_matrix(y, yhat)
    res = {}
    for i in range(cm.shape[0]):
        for j in range(cm.shape[1]):
            res[f"cm_{i:02}_{j:02}"] = cm[i][j]
    return res

def true_positive_rate(yhat: NDArray, y: NDArray):
    """TPR."""
    tp = (y * yhat).sum()
    fn = (y * (1 - yhat)).sum()
    return tp / (tp + fn)

def false_positive_rate(yhat: NDArray, y: NDArray):
    """FPR."""
    tn = ((1 - y) * (1 - yhat)).sum()
    fp = ((1 - y) * yhat).sum()
    return fp / (fp + tn)

def positive_predictive_value(yhat: NDArray, y: NDArray):
    """PPV."""
    tp = (y * yhat).sum()
    fp = ((1 - y) * yhat).sum()
    return tp / (tp + fp)

def positive_predictive_parity_ratio(yhat: NDArray, y: NDArray, a: NDArray):
    """Positive predictive parity ratio.
    Ratio of positive predictive value by protected group status
    """
    groups = np.unique(a)
    num_groups = len(groups)
    if num_groups < 2:
        return {}
    result = np.ones((num_groups))
    for i, group in enumerate(groups):
        ppv_a = positive_predictive_value(yhat[a == group], y[a == group])
        ppv_nota = positive_predictive_value(yhat[a != group], y[a != group])
        result[i] = ppv_a / ppv_nota
    return {f"positive_predictive_parity_ratio_{i:02}": v for i, v in enumerate(result)}

def theil_index(yhat: NDArray, y: NDArray, a: Optional[NDArray] = None):
    """Theil Index."""
    b = 1 + yhat - y
    ub = b.mean()
    return (np.log((b / ub) ** b) / ub).mean()

def theil_index_cm(tn: int, fp: int, fn: int, tp: int):
    """Theil Index from a confusion matrix."""
    bsum = tn + tp + (2 * fp)
    n = tn + fp + fn + tp
    ub = bsum / n
    btn = np.log((1 / ub)) / ub
    bfp = np.log((2 / ub) ** 2) / ub
    btp = btn
    return ((btn * tn) + (bfp * fp) + (btp * tp)) / n

def between_group_theil_index(yhat: NDArray, y: NDArray, a: NDArray):
    """Between group Theil index.
    Based on https://aif360.readthedocs.io/en/latest/modules/generated/aif360.metrics.ClassificationMetric.html#aif360.metrics.ClassificationMetric.between_group_generalized_entropy_index
    """
    groups = np.unique(a)
    b = np.zeros(len(groups), dtype=np.float64)
    for group in groups:
        y_pred = yhat[a == group]
        y_true = y[a == group]
        b[group] = np.mean(1 + y_pred - y_true)
    return np.mean(np.log((b / np.mean(b)) ** b) / np.mean(b))

def between_group_theil_index_cm(tn: NDArray, fp: NDArray, fn: NDArray, tp: NDArray):
    """Between group Theil index based on confusion matrix."""
    groups = np.arange(tn.shape[0])
    b = np.zeros(len(groups), dtype=np.float64)
    for g in groups:
        bsum = tn[g] + tp[g] + (2 * fp[g])
        n = tn[g] + fp[g] + fn[g] + tp[g]
        b[g] = bsum / n
    return np.mean(np.log((b / np.mean(b)) ** b) / np.mean(b))

def consistency_score(X: NDArray, yhat: NDArray, k: int = 5):
    """Consistency score. Measures that predictions are similar to nearest neighbors."""
    knn = NearestNeighbors(n_neighbors=k, algorithm="ball_tree").fit(X)
    idx = knn.kneighbors(X, return_distance=False)
    return 1 - abs(yhat - yhat[idx].mean(axis=1)).mean()

def add_binary_fairness_metrics(df: pd.DataFrame) -> pd.DataFrame:
    """Add fairness metrics to a data frame"""
    grouping = ["round"]
    if "group" in df.columns:
        grouping.append("group")
    if "cluster" in df.columns:
        grouping.append("cluster")
    if "dataset" in df.columns:
        grouping.append("dataset")
    if "strategy" in df.columns:
        grouping.append("strategy")
    if "trial" in df.columns:
        grouping.append("trial")
    if "concentration" in df.columns:
        grouping.append("concentration")
    dfp = df.pivot(index=grouping, columns="metric", values="value").reset_index()
    dfp = dfp.assign(
        val_ppv=ppv_cm(
            dfp["valid_cm_tn"], dfp["valid_cm_fp"], dfp["valid_cm_fn"], dfp["valid_cm_tp"]
        ),
        val_ppp=ppp_cm(
            dfp["valid_cm_tn"], dfp["valid_cm_fp"], dfp["valid_cm_fn"], dfp["valid_cm_tp"]
        ),
        val_tpr=tpr_cm(
            dfp["valid_cm_tn"], dfp["valid_cm_fp"], dfp["valid_cm_fn"], dfp["valid_cm_tp"]
        ),
        val_fpr=fpr_cm(
            dfp["valid_cm_tn"], dfp["valid_cm_fp"], dfp["valid_cm_fn"], dfp["valid_cm_tp"]
        ),
        val_theil=theil_index_cm(
            dfp["valid_cm_tn"], dfp["valid_cm_fp"], dfp["valid_cm_fn"], dfp["valid_cm_tp"]
        ),
    )
    if "group" in df.columns:
        if dfp.group.dtype.name == "category":
            groups = dfp.group.cat.categories
        else:
            groups = sorted(dfp.group.dropna().unique())
        for g in groups:
            # Calculate PPP ratio
            ingroup_ppv = dfp[dfp.group.eq(g)]["val_ppv"]
            outgroup = dfp[~dfp.group.eq(g) & ~dfp.group.isna()]
            outgroup_ppv = ppv_cm(
                outgroup["valid_cm_tn"],
                outgroup["valid_cm_fp"],
                outgroup["valid_cm_fn"],
                outgroup["valid_cm_tp"],
            )
            dfp.loc[dfp.group.eq(g), "ppp_ratio"] = ingroup_ppv.values / outgroup_ppv.values
            # Calculate Disparate Impact Ratio
            ingroup_ppp = dfp[dfp.group.eq(g)]["val_ppp"]
            outgroup_ppp = ppp_cm(
                outgroup["valid_cm_tn"],
                outgroup["valid_cm_fp"],
                outgroup["valid_cm_fn"],
                outgroup["valid_cm_tp"],
            )
            dfp.loc[dfp.group.eq(g), "disparate_impact_ratio"] = (
                ingroup_ppp.values / outgroup_ppp.values
            )
            # Calculate Equal Odds Ratio
            ingroup_tpr = dfp[dfp.group.eq(g)]["val_tpr"]
            ingroup_fpr = dfp[dfp.group.eq(g)]["val_fpr"]
            outgroup_tpr = tpr_cm(
                outgroup["valid_cm_tn"],
                outgroup["valid_cm_fp"],
                outgroup["valid_cm_fn"],
                outgroup["valid_cm_tp"],
            )
            outgroup_fpr = fpr_cm(
                outgroup["valid_cm_tn"],
                outgroup["valid_cm_fp"],
                outgroup["valid_cm_fn"],
                outgroup["valid_cm_tp"],
            )
            dfp.loc[dfp.group.eq(g), "disparate_impact_ratio"] = np.minimum(
                ingroup_tpr.values / outgroup_tpr.values,
                ingroup_fpr.values / outgroup_fpr.values,
            )
    return dfp
    # TODO: add fields for experiment parameters


#imports
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
import time
import re
import os
import sys

#Variables
checkurl = input("Enter URL to rate: ")
zulu = "https://zulu.zscaler.com/"
path = "C:\chromedriver.exe"
chrome = Service(path)
invalid = "invalid"
timeStr = time.strftime("%Y%m%d-%H%M%S")
from os.path import expanduser
home = expanduser("~")
line = '---------------------------------------'

#Chrome Driver Setup
#options.add_argument('window-size=1920x1080')
options = webdriver.ChromeOptions()
options.add_argument('headless')
options.add_argument("disable-gpu")
options.add_experimental_option('excludeSwitches', ['enable-logging'])
driver = webdriver.Chrome(service=chrome, options=options)

#Get ZULU
driver.get(zulu)

#Find URL text box and enter text
#checkurl = "test.com"
inputElement = driver.find_element(By.ID, "url")
inputElement.send_keys(checkurl)
#inputElement.send_keys("test.com")

#Find Analyze button and click
submit = driver.find_element(By.CSS_SELECTOR, '[value="Analyze"]')
submit.click()

#add Variable to Clipboard
def clipped(text):
    command = 'echo ' + text.strip() + '| clip'
    os.system(command)

#Logging output to Desktop
def logging():
	with open(home+'\\Desktop\\log.txt', "a") as external_file:
		add_text = timeStr, checkurl, score, isSafe, myURL
		print(add_text, file=external_file)
		external_file.close()

#Printing output to console
def printoutput():
	print(line)
	print("---------Zulu URL Risk Analyzer--------")
	print(line)
	print(timeStr, checkurl, score, isSafe, myURL)
	print(line)
	clipped(myURL)
	print("Reprot URL copied to Clipboard")
	print("Log entry created, "+home+'\\Desktop\\log.txt')
	print(line)

#validate page and fetch Reputation Score using logging()/printoutput()/restart()
while True:
	InvalidURL = driver.current_url
	if  invalid in InvalidURL or zulu == InvalidURL:
		driver.quit()
		print(line)
		print("Invalid URL - " + checkurl)
		print(line)
		print("Please make sure that you supplied valid url. (only ipv4 and FQDN)")
		print(line)
		os.execv(sys.executable, ['python'] + sys.argv)
		break
	else:
		rep = driver.find_element(By.ID, "rep-status").text
	if rep == "Completed":
		score = driver.find_element(By.ID, "rep-score").text
		score = re.sub('/100', '', score)
		#intScore = int(score)
		if int(score) >= 60:
			isSafe = "Fail"
		else:
			isSafe = "Pass"
		myURL = driver.current_url
		logging()
		printoutput()
		driver.quit()
		os.execv(sys.executable, ['python'] + sys.argv)
		break
	else:
		print("Analyzing...")
		time.sleep(2)


# -*- coding: utf-8 -*-

from zope.interface import implements
from collective.tablepage.interfaces import IFieldValidator
from ospfe.singole_procedure import _

class ValidatorCIG(object):
    """A string, max 10 chars"""
    implements(IFieldValidator)

    def __init__(self, field):
        self.field = field

    def validate(self, configuration, data=None):
        value = data or self.field.request.form.get(configuration['id'])
        if value and len(value)>10:
            return _('error_bad_cig', default='CIG must be a value of no more than 10 chars')

class ValidatorMax250Chars(object):
    """A string, max 250 chars"""
    implements(IFieldValidator)

    def __init__(self, field):
        self.field = field

    def validate(self, configuration, data=None):
        col_id = configuration['id']
        value = data or self.field.request.form.get(configuration['id'])
        if value and len(value)>250:
            return _('error_max_chars',
                     default='The value for "$name" must contain no more than 250 characters ($count provided).',
                     mapping={'name': configuration.get('label', col_id).decode('utf-8'),
                              'count': len(value)})


import requests
from bs4 import BeautifulSoup
from time import sleep
import pandas as pd
from timeit import default_timer as timer
from tqdm import tqdm
import random

# baselink = 'https://www.klenmarket.ru'
start = timer()

data = []
headers = {'User-Agent': 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G928X Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.83 Mobile Safari/537.36'}

with open("urlsAll.txt", "r") as file:
    linkList = file.readlines()
    # sleep(1)

for link in tqdm(linkList):
    r = requests.get(link.strip(), headers=headers, allow_redirects=False)
    sleep(random.randint(1, 5))
    if r.status_code != 200:
        continue
    # sleep(1)
    soup = BeautifulSoup(r.text, 'lxml')

    try:
        category1 = soup.findAll('a', class_='breadcrumbs__link')[1].text.strip().split(',')
    except:
        category1 = 'None'.split(',')
    try:
        category2 = soup.findAll('a', class_='breadcrumbs__link')[2].text.strip().split(',')
    except:
        category2 = 'None'.split(',')
    try:
        category3 = soup.findAll('a', class_='breadcrumbs__link')[3].text.strip().split(',')
    except:
        category3 = 'None'.split(',')
    try:
        category4 = soup.findAll('a', class_='breadcrumbs__link')[4].text.strip().split(',')
    except:
        category4 = 'None'.split(',')
    try:
        name = soup.find('h1').text.strip().split(',')
    except:
        name = 'None'.split(',')
    try:
        price = soup.find('span', class_='price__current-value').get('content')
    except:
        price = 'None'
    # try:
    #     code = soup.find('span', class_='origin__article').find('span', class_='text-warning').text.strip()
    # except:
    #     code = 'None'
    try:
        manufacturer = soup.find('div', class_='origin').find('a').text.strip()
    except:
        manufacturer = 'None'
    try:
        country = soup.find('span', class_='origin__country').text.strip()
    except:
        country = 'None'
    # link = 'https://stopgame.ru' + game.find('div', class_='caption').find('a').get('href')
    # score = items.find('div', class_='value').text
    # genre = items.find('div', class_='game-specs').findAll('div', class_='game-spec')[1].find('a').text
    data.append([' '.join(category1),
                 ' '.join(category2),
                 ' '.join(category3),
                 ' '.join(category4),
                 ' '.join(name),
                 price,
                 manufacturer,
                 country,
                 link.strip()])

# print(data)

#записываем полученные данные в CSV
header = ['category1',
          'category2',
          'category3',
          'category4',
          'name',
          'price',
          'manufacturer',
          'country',
          'link']
df = pd.DataFrame(data, columns=header)
df.to_csv(f'C:\\Users\d.novikov\PycharmProjects\Scrapers\Klenmarket.csv',sep=';', encoding='cp1251', errors='ignore')

end = timer()
print(end - start)


#import tensorflow as tf
import pandas as pd
from . import data_load_100hz
import numpy as np
from sklearn.preprocessing import LabelEncoder, OneHotEncoder

class Preprocess:
    def __init__(self):
        pass

    def delete_nan_from_data(self, data_100hz, label):
        indexes = []
        for index, data in enumerate(data_100hz):
            if 'Snow' not in data and 'Asphalt' not in data and 'Ice' not in data and 'Mu Change' not in data and 'Left = Asphalt, Right = Ice' not in data and 'Left = Ice, Right = Asphalt' not in data:
                indexes.append(index)
        data_100hz = np.array(data_100hz)

        deleted_100hz = np.delete(data_100hz, indexes, axis=0)

        deleted_100hz = np.delete(deleted_100hz, -1, axis=1)

        deleted_label = np.delete(np.array(label), indexes, axis=0)
        return deleted_100hz, deleted_label

    def delete_nan_from_100hz(self, data_100hz, label):
        indexes = []
        for index, data in enumerate(data_100hz):
            if 'Snow' not in data and 'Asphalt' not in data and 'Ice' not in data and 'Left = Ice, Right = Asphalt' not in data:
                indexes.append(index)
        data_100hz = np.array(data_100hz)
        deleted_100hz = np.delete(data_100hz, indexes, axis=0)

        deleted_100hz = np.delete(deleted_100hz, -1, axis=1)

        deleted_label = np.delete(np.array(label), indexes, axis=0)
        return deleted_100hz, deleted_label

    def one_hot_encoding(self, labels):
        le = LabelEncoder()
        result = []
        for label in labels:
            for l in label:
                result.append(l)
        le.fit(result)
        label_encoded = le.transform(result)

        oe = OneHotEncoder(sparse = False)
        integer_encoded = label_encoded.reshape(len(label_encoded), 1)
        onehot_encoded = oe.fit_transform(integer_encoded)

        return label_encoded, onehot_encoded


from flights.models import Airport, Airline, Agent, Itinerarie, Leg
from django.core.management.base import BaseCommand
from urllib.request import urlopen
import json
import datetime

URL = "https://raw.githubusercontent.com/Skyscanner/full-stack-recruitment-test/main/public/flights.json"
format = '%Y-%m-%dT%H:%M'

# python manage.py seed --mode=refresh

""" Clear all data and creates addresses """
MODE_REFRESH = 'refresh'

""" Clear all data and do not create any object """
MODE_CLEAR = 'clear'

class Command(BaseCommand):
    help = "seed database for testing and development."

    def add_arguments(self, parser):
        parser.add_argument('--mode', type=str, help="Mode")

    def handle(self, *args, **options):
        self.stdout.write('seeding data...')
        run_seed(self, options['mode'])
        self.stdout.write('done.')

def clear_data():
    """Deletes all the table data"""
    print('clearing data...')
    Leg.objects.all().delete()
    Itinerarie.objects.all().delete()

def create_data():
    """Creates an address object combining different elements from the list"""
    print("Creating data")
    response = urlopen(URL)
    data = json.loads(response.read().decode("utf-8"))
    itineraries = data['itineraries']
    legs = data['legs']

    for item in legs:

      # Create Airport object
      # Created = false if object exists
      departure_airport, dep_created = Airport.objects.get_or_create(id=item['departure_airport'])
      arrival_airport, arri_created  = Airport.objects.get_or_create(id=item['arrival_airport'])

      if dep_created:
        departure_airport.save()

      if arri_created:
        arrival_airport.save()

      departure_datetime = datetime.datetime.strptime(item['departure_time'], format)
      arrival_datetime = datetime.datetime.strptime(item['arrival_time'], format)

      # Craete Airline
      airline, air_created = Airline.objects.get_or_create(id = item['airline_id'], name = item['airline_name'])
      if air_created:
        airline.save()

      # Create Leg Object
      leg = Leg(
        id =  item['id'],
        departure_airport = departure_airport,
        arrival_airport = arrival_airport,
        departure_time = departure_datetime,
        arrival_time = arrival_datetime,
        stops = item['stops'],
        airline = airline,
        duration_mins = item['duration_mins']
      )
      leg.save()

    for item in itineraries:

      agent = Agent(agent = item['agent'],
                    agent_rating = item['agent_rating'])
      agent.save()

      itinerarie = Itinerarie(
        id = item['id'],
        price = item['price'],
        agent = agent
      )
      itinerarie.save()

      for item_leg in item['legs']:
        leg = Leg.objects.get(id=item_leg)
        itinerarie.legs.add(leg)

    print('Save Flights to database')
    return True

def run_seed(self, mode):
    """ Seed database based on mode
    :param mode: refresh / clear
    :return:
    """

    print('Running seed database')

    # Clear data from tables
    clear_data()
    if mode == MODE_CLEAR:
        return

    create_data()


#importing libraries and init program
import pygame
import sys
import os
from pygame.locals import *
pygame.init()

import random

#changes cwd to the project folder
os.chdir('..')

#fps value
fps = 30
frame_per_sec = pygame.time.Clock()

#colors
green = (0, 100, 0)
blue = (0, 153, 255)
white = (255, 255, 255)
red = (255, 0, 0)
black = (0, 0, 0)
yellow = (230, 184, 0)

#display w caption
display_surf = pygame.display.set_mode((1920,1080))
display_surf.fill(green)
pygame.display.set_caption("Blackjack")

#deck
deck = ["A", 2, 3, 4, 5, 6, 7, 8, 9, 10, "J", "Q", "K"]
#bust booleans
player_bust = False
dealer_bust = False
#stand booleans
player_stand = False
#quit boolean
quit = False
#deck values
player_deck = 0
dealer_deck = 0
#lists
player_list = []
dealer_list = []
#dealer positions
dealer_x = 740
dealer_y = 122
#player positions
player_x = 740
player_y = 662

#function to hit
def hit(person_deck, person_list, person, person_x, person_y):
    card = deck[random.randint(0,12)]
    if isinstance(card, str) and card!= "A":
        card_value = 10
        if "A" in person_list and (person_deck +10 > 21):
            card_value = 0
        person_list.append(card)
        image = display_card(card)
        display_surf.blit(image, (person_x, person_y))
        return card_value
    elif card == "A":
        if (person_deck + 11 > 21):
            person_list.append(card)
            hard_hand(person_list)
            card_value = 1
            image = display_card(card)
            display_surf.blit(image, (person_x, person_y))
            return card_value
        else:
            person_list.append(card)
            card_value = 11
            image = display_card(card)
            display_surf.blit(image, (person_x, person_y))
            return card_value
    else:
        person_list.append(card)
        if "A" in person_list and (person_deck + card > 21):
            hard_hand(person_list)
            card_value = (card - 10)
        else:
            card_value = card
        image = display_card(card)
        display_surf.blit(image, (person_x, person_y))
        return card

#changes the list once it changes to a hard hand
def hard_hand(person_list):
    for i in range(0, len(person_list)):
        if person_list[i] == "A":
            person_list[i] = "a"

#makes the card to image
def display_card(card):
    if card == 2:
        image = pygame.image.load(os.path.join(os.getcwd(), 'Assets', '2.jpg'))
        return image
    elif card == 3:
        image = pygame.image.load(os.path.join(os.getcwd(), 'Assets', '3.jpg'))
        return image
    elif card == 4:
        image = pygame.image.load(os.path.join(os.getcwd(), 'Assets', '4.jpg'))
        return image
    elif card == 5:
        image = pygame.image.load(os.path.join(os.getcwd(), 'Assets', '5.jpg'))
        return image
    elif card == 6:
        image = pygame.image.load(os.path.join(os.getcwd(), 'Assets', '6.jpg'))
        return image
    elif card == 7:
        image = pygame.image.load(os.path.join(os.getcwd(), 'Assets', '7.jpg'))
        return image
    elif card == 8:
        image = pygame.image.load(os.path.join(os.getcwd(), 'Assets', '8.jpg'))
        return image
    elif card == 9:
        image = pygame.image.load(os.path.join(os.getcwd(), 'Assets', '9.jpg'))
        return image
    elif card == 10:
        image = pygame.image.load(os.path.join(os.getcwd(), 'Assets', '10.jpg'))
        return image
    elif card == "J":
        image = pygame.image.load(os.path.join(os.getcwd(), 'Assets', 'J.jpg'))
        return image
    elif card == "Q":
        image = pygame.image.load(os.path.join(os.getcwd(), 'Assets', 'Q.jpq'))
        return image
    elif card == "K":
        image = pygame.image.load(os.path.join(os.getcwd(), 'Assets', 'K.jpg'))
        return image
    elif card == "A":
        image = pygame.image.load(os.path.join(os.getcwd(), 'Assets', 'A.jpg'))
        return image

class button():
    def __init__(self, color, x, y, width, height, text):
        self.color = color
        self.x = x
        self.y = y
        self.width = width
        self.height = height
        self.text = text

    def draw(self, win):
        pygame.draw.rect(win, black, (self.x-2, self.y-2, self.width+4, self.height+4), 0)
        #draws button to screen
        pygame.draw.rect(win, self.color, (self.x, self.y, self.width, self.height), 0)
        font = pygame.font.SysFont("trebuchet ms", 40)
        text = font.render(self.text, 1, (0,0,0))
        win.blit(text, (self.x + (self.width/2 - text.get_width()/2), self.y + (self.height/2 - text.get_height()/2)))

def is_clicked(x, y, width, height):
    if (pygame.mouse.get_pos()[0] >= x and pygame.mouse.get_pos()[0] <= (x + width)) and (pygame.mouse.get_pos()[1] >= y and pygame.mouse.get_pos()[1] <= (y + height)):
        return True

hit_button = button(blue, 806, 510, 130, 60, "HIT")
hit_button.draw(display_surf)

stand_button = button(yellow, 982, 510, 130, 60, "STAND")
stand_button.draw(display_surf)

exit_button = button(red, 60, 60, 60, 60, "X")
exit_button.draw(display_surf)

#Deal dealer's deck
image = pygame.image.load(r'C:\Users\charl\source\repos\blackjack3.0\Assets\template.jpg')
display_surf.blit(image, (dealer_x, dealer_y))
dealer_x += 240
dealer_deck += hit(dealer_deck, dealer_list, "dealer", dealer_x, dealer_y)
dealer_x +=240

#deal player's deck
for i in range(2):
    player_deck += hit(player_deck, player_list, "player", player_x, player_y)
    player_x += 240

while True:
    pygame.display.update()
    for event in pygame.event.get():
        #checks to see if the exit button was clicked
        #if event.type == QUIT:
        #    quit = True
        #    break
        #checks to see if the mouse clicked the hit button
        if pygame.mouse.get_pressed() == (1, 0, 0):
            if is_clicked(806, 510, 130, 60):
                #player hit
                player_deck += hit(player_deck, player_list, "player", player_x, player_y)
                player_x += 240
                if player_deck > 21:
                    pygame.display.update()
                    player_bust = True
                    break
        if pygame.mouse.get_pressed() == (1, 0, 0):
            if is_clicked(60, 60, 60, 60):
                quit = True
                #pygame.quit()
                break
        if pygame.mouse.get_pressed() == (1, 0, 0):
            if is_clicked(982, 510, 130, 60):
                player_stand = True
                break
    frame_per_sec.tick(fps)
    if quit == True:
        break
    if player_stand == True:
        break
    if player_bust == True:
        break

#dealer's turn
if quit == False:
    if player_bust == False:
        dealer_x -= 480
        dealer_deck += hit(dealer_deck, dealer_list, "dealer", dealer_x, dealer_y)
        dealer_x += 480
        while True:
            pygame.display.update()
            if dealer_deck < 17:
                dealer_deck += hit(dealer_deck, dealer_list, "dealer", dealer_x, dealer_y)
                dealer_x += 240
                pygame.time.delay(1000)
                if dealer_deck > 21:
                    pygame.display.update()
                    dealer_bust = True
                    break
            if dealer_deck == 17 and "A" in dealer_list:
                dealer_deck += hit(dealer_deck, dealer_list, "dealer", dealer_x, dealer_y)
                dealer_x += 240
                break
            if dealer_deck >= 17:
                break
            frame_per_sec.tick(fps)

#evaluates who won and creates a banner to say who won
if player_bust == True:
    end_button = button(red, 740, 464, 440, 152, "BUST")
    end_button.draw(display_surf)
elif dealer_bust == True:
    end_button = button(yellow, 740, 464, 440, 152, "DEALER BUSTED")
    end_button.draw(display_surf)
elif dealer_deck > player_deck and dealer_bust == False:
    end_button = button(red, 740, 464, 440, 152, "DEALER WINS")
    end_button.draw(display_surf)
elif player_deck > dealer_deck and player_bust == False:
    end_button = button(yellow, 740, 464, 440, 152, "PLAYER WINS")
    end_button.draw(display_surf)
elif player_deck == dealer_deck:
    end_button = button(white, 740, 464, 440, 152, "STAND")
    end_button.draw(display_surf)

#adds the end banner and checks to see if clicks exit button
if quit == False:
    while True:
        pygame.display.update()
        for event in pygame.event.get():
            #checks to see if the exit button was clicked
            if event.type == QUIT:
                quit = True
                break
            if pygame.mouse.get_pressed() == (1, 0, 0):
                if is_clicked(60, 60, 60, 60):
                    pygame.quit()
                    quit = True
                    break
        if quit == True:
            break
        frame_per_sec.tick(fps)


import streamlit as st
from streamlit.script_runner import RerunException
import pandas as pd
import calendar
pd.set_option("display.max_columns", None)
pd.set_option("mode.chained_assignment", None)
import plotly.express as px
from datetime import datetime
from dateutil.relativedelta import relativedelta

st.set_page_config(page_title="Bridgestone tools", layout="wide", page_icon="./materials/bs-logo-small.png")

# 設定不要顯示"Made with Stramlit"

hide_streamlit_style = """
            <style>
            #MainMenu {visibility: hidden;}
            footer {visibility: hidden;}
            </style>
            """
st.markdown(hide_streamlit_style, unsafe_allow_html=True)

### GFK數據處理視覺化

df = pd.read_csv(
    r"D:\kc.hsu\OneDrive - Bridgestone\GFK data\202201\GFK_CUSTOMREPORT_TRUCKTIRE_TW_BRIDGESTONE_Jan22.csv")

date = datetime.today().strftime("%Y-%m-%d")
date = datetime.strptime(date, "%Y-%m-%d")
interval = date - relativedelta(months=14)
interval = interval.strftime("%Y-%m")
gfk = df.copy()
gfk["Period"] = pd.to_datetime(gfk["Period"], yearfirst=True)
# gfk["Period"] = gfk["Period"].dt.to_period("M")
gfk["Period"] = gfk["Period"].dt.strftime("%Y-%m")
filtered_data = gfk[gfk["Period"] >= interval]

## Top 10 list
top10_players = ["BRIDGESTONE", "MAXXIS", "MICHELIN", "LINGLONG", "JINYUTIRES", "PRESA", "DUNLOP", "GOODYEAR",
                 "YOKOHAMA"]
filtered_data["PLAYERS"] = filtered_data["BRAND"].map(lambda x: "The others" if x not in top10_players else x)

filtered_data = filtered_data.astype({"SALES UNITS": "int", "SALES <LC>": "int", "PRICE TWD/UN.": "int"})

bar_sales_units = filtered_data.groupby(["Period", "BRIDGESTONE TYPE"])[["SALES UNITS"]].sum().reset_index()
bar_sales_units = bar_sales_units.sort_values(by="SALES UNITS", ascending=False)
bar_sales_units["in %"] = 100 * (
            bar_sales_units["SALES UNITS"] / bar_sales_units.groupby("Period")["SALES UNITS"].transform("sum"))

ttl_table = filtered_data[filtered_data["BRIDGESTONE TYPE"] != "EXCLUDED TBR AND LSR2"].groupby(["Period", "BRIDGESTONE TYPE"])[
    ["SALES UNITS"]].sum().unstack("Period").reset_index()

df = df.astype({"SALES UNITS": "int", "SALES <LC>": "int", "PRICE TWD/UN.": "int"})
df["Period"] = pd.to_datetime(df["Period"], dayfirst=True)
qt_data = df.groupby([pd.Grouper(key="Period", freq="QS"), "BRIDGESTONE TYPE"])[["SALES UNITS"]].sum().reset_index()
qt_data["Year"] = qt_data["Period"].dt.year
qt_data["Quarter"] = qt_data["Period"].dt.quarter
qt_data = qt_data.astype({"Year": "str", "Quarter": "str"})
qt_data["Y+Q"] = qt_data["Year"] + "Q" + qt_data["Quarter"]
qt_data["in %"] = 100 * (qt_data["SALES UNITS"] / qt_data.groupby("Period")["SALES UNITS"].transform("sum"))

ttl_qt_table = qt_data[qt_data["BRIDGESTONE TYPE"] != "EXCLUDED TBR AND LSR2"].groupby(["Y+Q", "BRIDGESTONE TYPE"])[["SALES UNITS"]].sum().unstack("Y+Q").reset_index()

### GFK文字敘述
gfk_numbers = filtered_data.groupby(["Period"])[["SALES UNITS"]].sum().reset_index()
previous_mon = int(datetime.today().strftime("%m")) - 1
before_pre_mon = previous_mon - 1
month_names = [calendar.month_name[i] for i in range(1, 13)]
months = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
month_dict = dict(zip(months, month_names))
result_description = "a. " + month_dict[previous_mon] + " vs " + month_dict[before_pre_mon] + " " + "{:.0f}%".format(100 * gfk_numbers.iloc[-1, 1] / gfk_numbers.iloc[-2, 1]) + " , vs PY " + "{:.0f}%".format(100 * gfk_numbers.iloc[-1, 1] / gfk_numbers.iloc[0, 1])

current_year = datetime.today().strftime("%Y")
# 檢查目前今年的資料是幾個月的
cum_months = len(gfk_numbers[gfk_numbers["Period"].str.contains(current_year)])
# 今年累計數量
ytd_units = gfk_numbers[gfk_numbers["Period"].str.contains(current_year)]["SALES UNITS"].sum()
# 去年累計數量
py_units = gfk_numbers[gfk_numbers["Period"].str.contains(str(cum_months - 1))]["SALES UNITS"][0:cum_months].sum()
ytd_py = "b. YTD vs PY " + "{:.0f}%".format(100 * ytd_units / py_units)

logo, space, header = st.columns([0.4, 0.4, 1])
with logo:
    st.image("./materials/bs-logo.png")

with header:
    st.header("GFK report")

st.subheader("Monthly Total sales in unit")

st.write(result_description)
st.write(ytd_py)
chart1, chart2 = st.columns([1, 0.6])
with chart1:
    st.plotly_chart(px.bar(bar_sales_units, x="Period", y="SALES UNITS", color="BRIDGESTONE TYPE",
                           text=bar_sales_units["in %"].map(lambda x: "{:.0f}%".format(x)),
                           title="Monthly truck tires sales units"))
with chart2:
    st.plotly_chart(px.bar(qt_data, x="Y+Q", y="SALES UNITS", color="BRIDGESTONE TYPE", text=qt_data["in %"].map(lambda x: "{:.0f}%".format(x)), labels={"Y+Q": "Quarter"}, title="Quarterly truck tires sales units"))

table1, table2 = st.columns([1, 0.6])
with table1:
    st.write(ttl_table)
with table2:
    st.write(ttl_qt_table)


from pyquery import PyQuery as pq
from proxypool.schemas.proxy import Proxy
from proxypool.crawlers.base import BaseCrawler
from loguru import logger
# import SimpleHTTPServer;
# import SocketServer;

BASE_URL = 'http://127.0.0.1:7777/proxyscrape_1000_http_proxies.txt'
# PORT = 7777; #server port
# Handler = SimpleHTTPServer.SimpleHTTPRequestHandler;
# httpd = SocketServer.TCPServer(("127.0.0.1", PORT), Handler);

class ProxyScrapeCrawler(BaseCrawler):
    """
    ipidea crawler, http://tiqu.linksocket.com:81/abroad?num=20&type=2&lb=1&sb=0&flow=1&regions=kr&n=0
    """
    urls = [BASE_URL]
    headers = {
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3534.4 Safari/537.36",
    }

    @logger.catch
    def crawl(self):
        """
        crawl main method
        """
        for url in self.urls:
            logger.info(f'fetching {url}')
            with open('/Users/anonymity/Desktop/COI/proxyscrape_1000_http_proxies.txt','r') as f:
                for proxy in self.parse(f):
                    logger.info(f'fetched proxy {proxy.string()} from {url}')
                    yield proxy
            f.close()

    def parse(self, file):
        """
        parse html file to get proxies
        :return:
        """
        try:
            proxies = file.readlines()
        except ValueError:
            print('cannot load the txt file from http://127.0.0.1:7777/proxyscrape_1000_http_proxies.txt')
            exit(-1)
        for proxy in proxies:
            print(proxy)
            strls = proxy.split(':')
            yield Proxy(host=strls[0].strip(), port=strls[1].strip())

if __name__ == '__main__':
    # httpd.serve_forever();
    crawler = ProxyScrapeCrawler()
    for proxy in crawler.crawl():
        print(proxy)


import pytest
import requests

resp= requests.get("https://reqres.in/api/users?page=2")
code = resp.status_code
assert code == 200, "code doesn't match"

json_resp = resp.json()
print(json_resp)

@pytest.mark.one
def test_json_resp():
    assert json_resp['total']==12
@pytest.mark.two
def test_json_resp_1():
    assert json_resp['total_pages'] == 2, "mismatch ib total pges"


import time
import numpy as np
import random
from PIL import Image, ImageOps, ImageFilter

import torch
import torch.optim as optim
import torch.nn as nn
import torch.utils.data as data
import torchvision.transforms as transforms

from utils.logger import AverageMeter
from utils import utils

class Barlow_Online:

	def __init__(self, te_query, te_gallery, model, checkpoint, img_transforms, args, device):

		self.te_query = te_query
		self.te_gallery = te_gallery
		self.model = model
		self.checkpoint = checkpoint

		self.transform, self.transform_prime = Transform_BT()
		self._eval_transform = img_transforms['eval']
		self.args = args
		self.device = device

		# normalization layer for the representations z1 and z2
		self.bn_layer = nn.BatchNorm1d(args.semantic_emb_size, affine=False).to(self.device)
		self.backbone_params = list(self.model.base_model.parameters())[:-2]
		self.classifier_params = self.model.base_model.last_linear.parameters()

		self.opt_net = optim.SGD(self.backbone_params, momentum=0, weight_decay=0, lr=self.args.lr_net)
		self.opt_clf = optim.SGD(list(self.classifier_params) + list(self.bn_layer.parameters()), momentum=self.args.momentum,
								 weight_decay=5e-4, nesterov=bool(self.args.nesterov), lr=self.args.lr_clf)

		self.bt_loss = AverageMeter()

	def adapt_single_sample(self, img):

		if self.args.online=='std':
			# print('000')
			self.model.load_state_dict(self.checkpoint['model_state_dict'])

		self.model.train()

		img_domain = img.split('/')[-3]
		if img_domain=='sketch' or img_domain=='quickdraw':
			sample = ImageOps.invert(Image.open(img)).convert(mode='RGB')
		else:
			sample = Image.open(img).convert(mode='RGB')

		for _ in range(self.args.num_iter):
			x1 = [self.transform(sample) for _ in range(self.args.batch_size_train)]
			x2 = [self.transform_prime(sample) for _ in range(self.args.batch_size_train)]

			im1 = torch.stack(x1)
			im2 = torch.stack(x2)

			im1 = im1.float().to(self.device)
			im2 = im2.float().to(self.device)

			self.opt_net.zero_grad()
			self.opt_clf.zero_grad()

			_, im_feat1 = self.model(im1)
			_, im_feat2 = self.model(im2)

			z1 = self.model.base_model.last_linear(im_feat1)
			z2 = self.model.base_model.last_linear(im_feat2)

			# z1 = projector(z1)
			# z2 = projector(z2)

			# empirical cross-correlation matrix
			# c = torch.t(z1) @ z2
			c = torch.t(self.bn_layer(z1)) @ self.bn_layer(z2)
			c.div_(self.args.batch_size_train)

			on_diag = torch.diagonal(c).add_(-1).pow_(2).sum()
			off_diag = off_diagonal(c).pow_(2).sum()
			loss = on_diag + self.args.lambd * off_diag
			loss.backward()

			self.opt_net.step()
			self.opt_clf.step()

			self.bt_loss.update(loss.item(), im1.size(0))

	def test_single_sample(self, img):

		self.model.eval()

		img_domain = img.split('/')[-3]
		img_clss = img.split('/')[-2]

		if img_domain=='sketch' or img_domain=='quickdraw':
			sample = ImageOps.invert(Image.open(img)).convert(mode='RGB')
		else:
			sample = Image.open(img).convert(mode='RGB')

		with torch.no_grad():
			inputs = self._eval_transform(sample).unsqueeze(0).to(self.device)

			_, im_feat = self.model(inputs)
			im_em = self.model.base_model.last_linear(im_feat)

		return im_em.cpu().data.numpy(), np.expand_dims(np.array(img_clss), axis=0)

	def save_model(self, path_cp, model_save_name):

		utils.save_checkpoint({
							'iter':self.args.num_iter,
							'model_state_dict':self.model.state_dict(),
							}, directory=path_cp, save_name=model_save_name, last_chkpt='')

	def train(self):

		# Start counting time
		start = time.time()

		np.random.shuffle(self.te_query['te'])

		print('\nQuery set size: ', len(self.te_query['te']))
		for i in range(len(self.te_query['te'])):

			self.adapt_single_sample(self.te_query['te'][i])
			sk_em, cls_sk = self.test_single_sample(self.te_query['te'][i])

			# Accumulate sketch embedding
			if i == 0:
				self.acc_sk_em = sk_em
				self.acc_cls_sk = cls_sk
			else:
				self.acc_sk_em = np.concatenate((self.acc_sk_em, sk_em), axis=0)
				self.acc_cls_sk = np.concatenate((self.acc_cls_sk, cls_sk), axis=0)

			if (i+1) % self.args.log_interval == 0:
				print('[Train] [{0}/{1}]\t'
					  'BT loss: {rn.val:.4f} ({rn.avg:.4f})\t'
					  .format(i+1, len(self.te_query['te']), rn=self.bt_loss))

			# if (i+1)==50:
			# 	break

		if self.args.include_gallery:
			np.random.shuffle(self.te_gallery['te_unseen_cls'])

		print('\nGallery set size: ', len(self.te_gallery['te_unseen_cls']))
		for i in range(len(self.te_gallery['te_unseen_cls'])):

			if self.args.include_gallery:
				self.adapt_single_sample(self.te_gallery['te_unseen_cls'][i])

			im_em, cls_im = self.test_single_sample(self.te_gallery['te_unseen_cls'][i])

			# Accumulate image embedding
			if i == 0:
				self.acc_im_em = im_em
				self.acc_cls_im = cls_im
			else:
				self.acc_im_em = np.concatenate((self.acc_im_em, im_em), axis=0)
				self.acc_cls_im = np.concatenate((self.acc_cls_im, cls_im), axis=0)

			if (i+1) % self.args.log_interval == 0:
				print('[Train] [{0}/{1}]\t'
					  'BT loss: {rn.val:.4f} ({rn.avg:.4f})\t'
					  .format(i+1, len(self.te_gallery['te_unseen_cls']), rn=self.bt_loss))

			# if (i+1)==50:
			# 	break

		if self.args.dataset=='DomainNet':
			print('\nSeen gallery set size: ', len(self.te_gallery['te_seen_cls']))
			for i in range(len(self.te_gallery['te_seen_cls'])):

				im_seen_em, cls_seen_im = self.test_single_sample(self.te_gallery['te_seen_cls'][i])

				# Accumulate image embedding
				if i == 0:
					acc_im_seen_em = im_seen_em
					acc_cls_seen_im = cls_seen_im
				else:
					acc_im_seen_em = np.concatenate((acc_im_seen_em, im_seen_em), axis=0)
					acc_cls_seen_im = np.concatenate((acc_cls_seen_im, cls_seen_im), axis=0)

				# if (i+1)==50:
				# 	break

			self.acc_im_em_gzs = np.concatenate((self.acc_im_em, acc_im_seen_em), axis=0)
			self.acc_cls_im_gzs = np.concatenate((self.acc_cls_im, acc_cls_seen_im), axis=0)

			print('\nSeen + Unseen Gallery Emb Dim:{}'.format(self.acc_im_em_gzs.shape))

		print('\nQuery Emb Dim:{}; Gallery Emb Dim:{}.'.format(self.acc_sk_em.shape, self.acc_im_em.shape))

		end = time.time()
		elapsed = end-start
		print(f"Time Taken:{elapsed//60:.0f}m{elapsed%60:.0f}s.\n")

def barlow_ttt(loader, model, args):

	# normalization layer for the representations z1 and z2
	bn_layer = nn.BatchNorm1d(args.semantic_emb_size, affine=False).cuda()

	backbone_params = list(model.base_model.parameters())[:-2]
	classifier_params = model.base_model.last_linear.parameters()

	opt_net = optim.SGD(backbone_params, momentum=0, weight_decay=0, lr=args.lr_net)
	opt_clf = optim.SGD(list(classifier_params) + list(bn_layer.parameters()), momentum=0.9, weight_decay=5e-4, nesterov=True, lr=args.lr_clf)

	model.train()
	bt_loss = AverageMeter()

	for epoch in range(args.epochs):

		# Start counting time
		start = time.time()

		for i, (im1, im2, _) in enumerate(loader):

			im1 = im1.float().cuda()
			im2 = im2.float().cuda()

			opt_net.zero_grad()
			opt_clf.zero_grad()

			_, im_feat1 = model(im1)
			_, im_feat2 = model(im2)

			z1 = model.base_model.last_linear(im_feat1)
			z2 = model.base_model.last_linear(im_feat2)

			# z1 = projector(z1)
			# z2 = projector(z2)

			# empirical cross-correlation matrix
			# c = torch.t(z1) @ z2
			c = torch.t(bn_layer(z1)) @ bn_layer(z2)
			c.div_(args.batch_size)

			on_diag = torch.diagonal(c).add_(-1).pow_(2).sum()
			off_diag = off_diagonal(c).pow_(2).sum()
			loss = on_diag + args.lambd*off_diag
			loss.backward()

			opt_net.step()
			opt_clf.step()

			bt_loss.update(loss.item(), im1.size(0))

			if (i+1) % args.log_interval == 0:
				print('[Train] Epoch: [{0}][{1}/{2}]\t'
					  'BT loss: {bt.val:.4f} ({bt.avg:.4f})\t'
					  .format(epoch+1, i+1, len(loader), bt=bt_loss))

		end = time.time()
		elapsed = end-start
		print(f"Time Taken:{elapsed//60:.0f}m{elapsed%60:.0f}s.\n")

	return model

class BarlowDataset(data.Dataset):
	def __init__(self, fls, transform, transform_prime):

		self.fls = fls
		self.clss = np.array([f.split('/')[-2] for f in fls])
		self.domains = np.array([f.split('/')[-3] for f in fls])
		self.transform = transform
		self.transform_prime = transform_prime

	def __getitem__(self, item):

		sample_domain = self.domains[item]
		if sample_domain=='sketch' or sample_domain=='quickdraw':
			sample = ImageOps.invert(Image.open(self.fls[item])).convert(mode='RGB')
		else:
			sample = Image.open(self.fls[item]).convert(mode='RGB')
		# sample = Image.open(self.fls[item]).convert(mode='RGB')

		clss = self.clss[item]

		x1 = self.transform(sample)
		x2 = self.transform_prime(sample)

		return x1, x2, clss

	def __len__(self):
		return len(self.fls)

def off_diagonal(x):
	# return a flattened view of the off-diagonal elements of a square matrix
	n, m = x.shape
	assert n == m
	return x.flatten()[:-1].view(n - 1, n + 1)[:, 1:].flatten()

class GaussianBlur(object):
	def __init__(self, p):
		self.p = p

	def __call__(self, img):
		if random.random() < self.p:
			sigma = random.random() * 1.9 + 0.1
			return img.filter(ImageFilter.GaussianBlur(sigma))
		else:
			return img

class Solarization(object):
	def __init__(self, p):
		self.p = p

	def __call__(self, img):
		if random.random() < self.p:
			return ImageOps.solarize(img)
		else:
			return img

def Transform_BT():

	# Imagenet standards
	im_mean = [0.485, 0.456, 0.406]
	im_std = [0.229, 0.224, 0.225]

	transform = transforms.Compose([
		transforms.RandomResizedCrop(224, interpolation=Image.BICUBIC),
		transforms.RandomHorizontalFlip(p=0.5),
		transforms.RandomApply([transforms.ColorJitter(brightness=0.4, contrast=0.4, saturation=0.2, hue=0.1)], p=0.8),
		transforms.RandomGrayscale(p=0.2),
		GaussianBlur(p=1.0),
		Solarization(p=0.0),
		transforms.ToTensor(),
		transforms.Normalize(im_mean, im_std)
	])

	transform_prime = transforms.Compose([
		transforms.RandomResizedCrop(224, interpolation=Image.BICUBIC),
		transforms.RandomHorizontalFlip(p=0.5),
		transforms.RandomApply([transforms.ColorJitter(brightness=0.4, contrast=0.4, saturation=0.2, hue=0.1)], p=0.8),
		transforms.RandomGrayscale(p=0.2),
		GaussianBlur(p=0.1),
		Solarization(p=0.2),
		transforms.ToTensor(),
		transforms.Normalize(im_mean, im_std)
	])

	return transform, transform_prime


import re
import os
from email.parser import Parser
import argparse
import shutil

class EmlDisassembly:
    def __init__(self):
        self.list_paths = []

    def message_count(self, path_file):
        for (paths,dirs,files) in os.walk(path_file):
            for file in files:
                path = paths + file
                self.list_paths.append(path)
        messages = '\n\033[47m\033[30m----Number of messages in a folder:\033[0m \033[32m{0}\033[0m\n'.format(str(len(self.list_paths)))
        print(messages)

    def movefile(self,path,login,file):
        shutil.copyfile(path, 'downloads\\' + login + '\\' + file)
        message = '----Message: {0}'.format(file)
        print(message)

    def makedir(self,login):
        os.mkdir('downloads\\' + login)
        print('\033[30m---Create folder with name---{0}\033[0m'.format(login))

    def check_downloads_folder(self):
        dirs = [dirs for (paths,dirs,files) in os.walk('.')]
        if dirs[0].count('downloads'):
            message = '\033[32mFolder <DOWNLOADS> exists\033[0m'
            print(message)
        else:
            os.mkdir('downloads')
            message = '\033[32mFolder <DOWNLOADS> make\033[0m'
            print(message)

    def folder_mails(self, path_file):
        for (paths,dirs,files) in os.walk(path_file):
            for file in files:
                path = paths + file
                with open(path) as fp:
                    emailcontent = Parser().parsestr(fp.read())
                    To = emailcontent['To']
                    with open('logins.txt') as logins_file:
                        for login_file in logins_file.readlines():
                            login_up = ''.join(login_file.split('\n'))
                            if re.findall(login_up, To):
                                try:
                                    self.makedir(login_up)
                                    self.movefile(path, login_up, file)
                                except OSError:
                                    self.movefile(path, login_up, file)
                            else:
                                pass

    def main(self):
        try:
            self.check_downloads_folder()
            self.message_count('files\\')
            self.folder_mails('files\\')
        except TypeError:
            pass

if __name__ == '__main__':
    obj = EmlDisassembly()
    obj.main()


"""
Modeling a Two Tank system with neural ODEs
"""

import torch
import numpy as np
import slim
import psl

from neuromancer import blocks, estimators, dynamics, arg, integrators, ode
from neuromancer.interpolation import LinInterp_Offline
from neuromancer.activations import activations
from neuromancer.visuals import VisualizerOpen
from neuromancer.trainer import Trainer
from neuromancer.problem import Problem
from neuromancer.simulators import OpenLoopSimulator, MultiSequenceOpenLoopSimulator
from neuromancer.callbacks import SysIDCallback
from neuromancer.loggers import BasicLogger, MLFlowLogger
from neuromancer.dataset import get_sequence_dataloaders
from neuromancer.constraint import variable
from neuromancer.loss import PenaltyLoss

torch.manual_seed(0)
device = "cpu"

system = psl.systems['TwoTank']

ts = 1.0
nsim = 2000
modelSystem = system(ts=ts, nsim=nsim)
raw = modelSystem.simulate()
psl.plot.pltOL(Y=raw['Y'])
psl.plot.pltPhase(X=raw['Y'])

t = (np.arange(nsim+1)*ts).reshape(-1, 1)
raw['Time'] = t[:-1]

#%% Interpolant for offline mode
t = torch.from_numpy(t)
u = raw['U'].astype(np.float32)
u = np.append(u,u[-1,:]).reshape(-1,2)
ut = torch.from_numpy(u)
interp_u = LinInterp_Offline(t, ut)

#%% Interpolation class for online mode
# Interp_Online = LinInterp_Online()

#%%
#  Train, Development, Test sets - nstep and loop format
nsteps = 30  # nsteps rollouts in training
nstep_data, loop_data, dims = get_sequence_dataloaders(raw, nsteps,
                            moving_horizon=True)

train_data, dev_data, test_data = nstep_data
train_loop, dev_loop, test_loop = loop_data

# %% Identity mapping
nx = dims['X'][1]

estim = estimators.FullyObservable(
    {**train_data.dataset.dims, "x0": (nx,)},
    linear_map=slim.maps['identity'],
    input_keys=["Yp"],
)

estim(train_data.dataset.get_full_batch())

# %% Instantiate the blocks, dynamics model:
nu = dims['U'][1]
black_box_ode = blocks.MLP(insize=nx+nu, outsize=nx, hsizes=[128, 128, 128],
                           linear_map=slim.maps['linear'],
                           nonlin=activations['gelu'])
fx_int = integrators.RK4(black_box_ode, interp_u=interp_u, h=modelSystem.ts)
fy = slim.maps['identity'](nx, nx)

dynamics_model = dynamics.ODENonAuto(fx_int, fy, extra_inputs=['Uf'],
                input_key_map={"x0": f"x0_{estim.name}", "Time": "Timef", 'Yf': 'Yf'},  #TBC2: sth wrong with input_key_map
                name='dynamics',  # must be named 'dynamics' due to some issue in visuals.py
                online_flag=False
                )

# %% Constraints + losses:
yhat = variable(f"Y_pred_{dynamics_model.name}")
y = variable("Yf")

yFD = (y[:, 1:, :] - y[:, :-1, :])
yhatFD = (yhat[:, 1:, :] - yhat[:, :-1, :])

fd_loss = 2.0*((yFD == yhatFD)^2)
fd_loss.name = 'FD_loss'

reference_loss = ((yhat == y)^2)
reference_loss.name = "ref_loss"

# %%
objectives = [reference_loss, fd_loss]
constraints = []
components = [estim, dynamics_model]
# create constrained optimization loss
loss = PenaltyLoss(objectives, constraints)
# construct constrained optimization problem
problem = Problem(components, loss)
# plot computational graph
problem.plot_graph()
problem = problem.to(device)

# %%
optimizer = torch.optim.Adam(problem.parameters(), lr=0.001)
logger = BasicLogger(args=None, savedir='test', verbosity=1,
                     stdout="nstep_dev_"+reference_loss.output_keys[0])

simulator = OpenLoopSimulator(
    problem, train_loop, dev_loop, test_loop, eval_sim=True, device=device,
) if isinstance(train_loop, dict) else MultiSequenceOpenLoopSimulator(
    problem, train_loop, dev_loop, test_loop, eval_sim=True, device=device,
)
visualizer = VisualizerOpen(
    dynamics_model,
    1,
    'test',
    training_visuals=False,
    trace_movie=False,
)
callback = SysIDCallback(simulator, visualizer)

trainer = Trainer(
    problem,
    train_data,
    dev_data,
    test_data,
    optimizer,
    callback=callback,
    patience=100,
    warmup=10,
    epochs=150,
    eval_metric="nstep_dev_"+reference_loss.output_keys[0],
    train_metric="nstep_train_loss",
    dev_metric="nstep_dev_loss",
    test_metric="nstep_test_loss",
    logger=logger,
    device=device,
)
# %%
best_model = trainer.train()
# %%
best_outputs = trainer.test(best_model)
# %%


import urllib.request
import csv
import json
import os.path
import string
import time
import pandas as pd
import numpy as np
from sklearn import preprocessing
from sklearn import model_selection
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix
from sklearn.metrics import accuracy_score
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.naive_bayes import GaussianNB
from sklearn.svm import SVC

# Runescape item category names
cat_names = ["Miscellaneous","Ammo", "Arrows", "Bolts", "Construction_Materials"
, "Construction_Projects", "Cooking_Ingredients","Costumes"
,"Crafting_Materials","Familiars","Farming_Produce","Fletching_Materials"
,"Food_and_Drink","Herblore_Materials","Hunting_Equipment","Hunting_Produce"
,"Jewellery","Mage_Armour","Mage_Weapons", "Melee_Armor_Low_Level","Melee_Armor_Mid_Level",
"Melee_Armor_High_Level","Melee_Weapons_Low_Level","Melee_Weapons_Mid_Level","Melee_Weapons_High_Level","Mining_and_Smithing","Potions","Prayer_Armour","Prayer_Materials",
"Range_Armour","Range_Weapons","Runecrafting","Runes_Spells_Teleports","Seeds","Summoning_Scrolls",
"Tools_and_Containers","Woodcutting_Products","Pocket_Items"]

def get_data_from_row(row, m, n):
    arr = list(row)
    # NORMALIZING DATA FROM ARR
    minn = min(arr)
    for x in range(len(arr)):
        arr[x] = arr[x] - minn
    maxx = max(arr)
    for x in range(len(arr)):
        if arr[x] != 0:
            arr[x] = maxx / arr[x]
    # NORMALIZING DATA FROM ARR
    row_X = []
    row_Y = []
    true_set = set()
    # Predict if price increases over the next n days based on the previous m
    for x in range(0, len(arr)-m-n, m+n+1):
        past_m = list(arr[x:x+m])
        row_X.append(past_m)
        next_n = True if arr[x+m] < arr[x+m+n] else False # True if price increased over next n days
        row_Y.append(next_n)
        true_set.add(next_n)
    # Making sure all data points aren't in the same class, not useful for ML
    if len(true_set) == 1:
        return [], []
    return row_X, row_Y
def test_models(X, Y):
    models = []
    models.append(('LogisticRegression', LogisticRegression(solver='liblinear', multi_class='ovr')))
    models.append(('LinearDiscriminantAnalysis', LinearDiscriminantAnalysis()))
    models.append(('KNeighborsClassifier', KNeighborsClassifier()))
    models.append(('DecisionTreeClassifier', DecisionTreeClassifier()))
    models.append(('GaussianNB', GaussianNB()))
    # SVM problems with more than 200,000 data points
    n_splits = 10 # Number of k-fold splits, decrease this number if it doesn't work due to large m + n size
    seed = 7
    scoring = "accuracy"
    results = []
    names = []
    for name, model in models:
        kfold = model_selection.KFold(n_splits=n_splits, random_state=seed)
        cv_results = model_selection.cross_val_score(model, X, Y, cv=kfold, scoring=scoring)
        results.append(cv_results)
        names.append(name)
        msg = "%s: %f (%f)" % (name, cv_results.mean(), cv_results.std())
        print(msg)
def get_cat_data(headers, cat, m, n):
    df = pd.read_csv("item_graphs/"+cat+".csv", names=headers, index_col = 0)
    cat_X = []
    cat_Y = []
    for index, row in df.iterrows():
        row_X, row_Y = get_data_from_row(row, m, n)
        cat_X.extend(row_X)
        cat_Y.extend(row_Y)
    return cat_X, cat_Y
# machine_learning(m, n): Tests models for ML, price increase in next n days based on previous m days of data
def machine_learning(m, n):
    headers = ["Name"] + [x for x in range(-179, 1)]
    all_X = []
    all_Y = []
    for cat in cat_names:
        print("Data for "+cat+" Category: (" +str(m) + ","+str(n)+")")
        cat_X, cat_Y = get_cat_data(headers, cat, m, n)
        all_X.extend(cat_X)
        all_Y.extend(cat_Y)
    print("Testing Models:")
    test_models(all_X, all_Y)

machine_learning(4,2)
# Run with python3 -W ignore machine_learning.py  to suppress warnings about collinear variables