import codecs
import json
import logging
import re
from typing import Optional

import requests
from bs4 import BeautifulSoup

from argostime.exceptions import CrawlerException
from argostime.exceptions import PageNotFoundException

from argostime.crawler.crawl_utils import CrawlResult, register_crawler

@register_crawler("HEMA", "hema.nl")
def crawl_hema(url: str) -> CrawlResult:
    """Crawler for hema.nl"""

    response: requests.Response = requests.get(url, timeout=10)

    if response.status_code != 200:
        logging.error("Got status code %d while getting url %s", response.status_code, url)
        raise PageNotFoundException(url)

    soup = BeautifulSoup(response.text, "html.parser")

    result: CrawlResult = CrawlResult(url=url)

    matches = soup.find_all("script")
    pattern = re.compile(r"var gtmDataObj = JSON\.parse\(.+\);")

    for match in matches:
        re_result: Optional[re.Match] = pattern.search(match.text)

        if re_result is not None:
            raw_json = re_result.group(0)
            logging.debug("Found raw product json %s", raw_json)
            break

    raw_json = raw_json.replace("var gtmDataObj = JSON.parse('", "")
    raw_json = raw_json.replace("');", "")

    raw_json = codecs.decode(raw_json, "unicode-escape")

    try:
        product_dict = json.loads(raw_json)
    except json.decoder.JSONDecodeError as exception:
        logging.error("Could not decode JSON %s, raising CrawlerException", raw_json)
        raise CrawlerException from exception

    logging.debug(product_dict)

    try:
        result.product_name = product_dict["ecommerce"]["detail"]["products"][0]["name"]
    except KeyError as exception:
        logging.error("Could not find product name in %s via %s", raw_json, url)
        raise CrawlerException from exception

    try:
        result.product_code = product_dict["ecommerce"]["detail"]["products"][0]["id"]
    except KeyError as exception:
        logging.error("Could not find product code in %s via %s", raw_json, url)
        raise CrawlerException from exception

    try:
        result.normal_price = float(product_dict["ecommerce"]["detail"]["products"][0]["price"])
    except KeyError as exception:
        logging.error("Could not find a valid price in %s via %s", raw_json, url)
        result.normal_price = -1

    return result


from rasa.nlu.config import RasaNLUModelConfig
from rasa.shared.nlu.constants import TEXT
from rasa.shared.nlu.training_data.message import Message

def test_spacy_ner_extractor(component_builder, spacy_nlp):
    _config = RasaNLUModelConfig({"pipeline": [{"name": "SpacyEntityExtractor"}]})
    ext = component_builder.create_component(_config.for_component(0), _config)
    example = Message(
        data={
            TEXT: "anywhere in the U.K.",
            "intent": "restaurant_search",
            "entities": [],
            "text_spacy_doc": spacy_nlp("anywhere in the west"),
        }
    )

    ext.process(example, spacy_nlp=spacy_nlp)

    assert len(example.get("entities", [])) == 1
    assert example.get("entities")[0] == {
        "start": 16,
        "extractor": "SpacyEntityExtractor",
        "end": 20,
        "value": "U.K.",
        "entity": "GPE",
        "confidence": None,
    }

    # Test dimension filtering includes only specified dimensions

    example = Message(
        data={
            TEXT: "anywhere in the West with Sebastian Thrun",
            "intent": "example_intent",
            "entities": [],
            "text_spacy_doc": spacy_nlp("anywhere in the West with Sebastian Thrun"),
        }
    )
    _config = RasaNLUModelConfig({"pipeline": [{"name": "SpacyEntityExtractor"}]})

    _config.set_component_attr(0, dimensions=["PERSON"])
    ext = component_builder.create_component(_config.for_component(0), _config)
    ext.process(example, spacy_nlp=spacy_nlp)

    assert len(example.get("entities", [])) == 1
    assert example.get("entities")[0] == {
        "start": 26,
        "extractor": "SpacyEntityExtractor",
        "end": 41,
        "value": "Sebastian Thrun",
        "entity": "PERSON",
        "confidence": None,
    }


from email.mime import audio
import os
from pdb import Restart
from pydoc import text
from re import M
import time
from tkinter.messagebox import YES
from turtle import reset
import playsound
import speech_recognition as sr
from gtts import gTTS
import tkinter as tk
from tkinter import ttk
from tkinter.ttk import *

def speak(text):
    tts = gTTS(text=text, lang="en")
    filename = "voice1.mp3"
    tts.save(filename)
    playsound.playsound(filename)

def get_audio():
    r = sr.Recognizer()
    with sr.Microphone() as source:
        audio = r.listen(source)
        said = ""

        try:
            said = r.recognize_google(audio)
            print(said)
        except Exception as e:
            print("Exception: " + str(e))

    return said
def write_slogan():
    text = get_audio()
    dir = "C:/Program Files/Mozilla Firefox/firefox.exe"
    photoshop = "C:/Program Files/Adobe/Adobe Photoshop 2021/photoshop.exe"
    chrome = "C:/Program Files/Google/Chrome/Application/chrome.exe"
    vscode = "C:/Users/Milad/AppData/Local/Programs/Microsoft VS Code/Code.exe"
    word = "C:/Program Files/Microsoft Office/root/Office16/winword.exe"
    exel = "C:/Program Files/Microsoft Office/root/Office16/EXCEL.EXE"
    access = "C:/Program Files/Microsoft Office/root/Office16/MSACCESS.EXE"
    powerpoint = "C:/Program Files/Microsoft Office/root/Office16/POWERPNT.EXE"

    # for hey in text:
    #     speak("Yes, I am in service")
    #     break
    if "hello" in text:
        speak("hello Iam tkinter, how are you ?")
        write_slogan()
    elif "how are you" in text:
        speak("Thanks, i am great")
        write_slogan()
    elif "thanks" in text :
        speak("it is Good")
    elif "iam find, Whats up ?" in text:
        speak("Thanks everything is great")
    elif "what's your name" in text:
        speak("My name is bam")
    elif "please open the Firefox" in text:
        speak("ok, please wait")
        os.startfile(dir)
    elif "please open the Photo Shop" in text:
        speak("ok, please wait")
        os.startfile(photoshop)
    elif "please open the Chrome" in text:
        speak("ok, please wait")
        os.startfile(chrome)
    elif "please open the vs code" in text:
        speak("ok, please wait")
        os.startfile(vscode)
    elif "please open the word" in text:
        speak("ok, please wait")
        os.startfile(word)
    elif "please open the Excel" in text:
        speak("ok, please wait")
        os.startfile(exel)
    elif "please open the access" in text:
        speak("ok, please wait")
        os.startfile(access)
    elif "please open the PowerPoint" in text:
        speak("ok, please wait")
        os.startfile(powerpoint)
    else:
        speak("Invalid input")
        print("Invalid input")

root = tk.Tk()
frame = tk.Frame(root)
frame.pack()
root.geometry("200x300")
style = Style()

style.configure('TButton', font =
			('calibri', 20, 'bold'),
					borderwidth = '4')
style.map('TButton', foreground = [('active', '!disabled', 'green')],
					background = [('active', 'black')])
btn1 = Button(frame,
                   text="Send Voice",
                   command=write_slogan)
btn1.grid(row = 0, column = 3, padx = 0)
root.mainloop()


import requests
import time
import sys

print('''
    #############################
    #   *         *         *   #
    #   *   ¡Paginas web    *   #
    #   *         *         *   #
    #############################
''')

url = input("pagina a refrescar: ")
# ejemplo 60 = 1min, 120 = 2min etc...
refrescar = int(input("cada cuanto quiere que se actualize (segundos): "))

while True:
    responde = requests.get(url)
    if responde.status_code == 200:
        print(f"Contenido actualizado:\n{responde.text}")
        print(f"pagina actualizada se refrescara en {refrescar} segundos")
    else:
        print(f"No se pudo obtener el contenido (Código de estado: {responde.status_code})")
        print("no se actulizo la pagina")
        res = input("salir S o N").lower()
        if res == "s":
            sys.exit()
        elif res == "n":
            print(f"la pagina se refrescara en {refrescar}")

    time.sleep(refrescar)


from flask import Flask, render_template, request, redirect
import os
from flask_sqlalchemy import SQLAlchemy

pathdir = os.path.dirname(os.path.abspath(__file__))
db_uri = 'sqlite:///' + os.path.join(pathdir, 'DB.sqlite')

app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = db_uri

db = SQLAlchemy(app)

class Teilnehmer(db.Model):

    member = db.Column(db.String(80), primary_key=True, unique=True)

@app.route('/', methods=['POST', 'GET'])
def home():
    if request.form:

        new_Teilnehmer = Teilnehmer(member=request.form.get('member'))
        db.session.add(new_Teilnehmer)
        db.session.commit()

    x = Teilnehmer.query.all()

    return(render_template('home.html', member_list=x))

@app.route('/update', methods=['POST'])
def update():
    newtitle = request.form.get('newtitle')
    oldtitle = request.form.get('oldtitle')
    kandidat = Teilnehmer.query.filter_by(member=oldtitle).first()
    kandidat.member = newtitle
    db.session.commit()
    return(redirect('/'))

if __name__ == '__main__':
    app.run(debug=True)


from datetime import timedelta

from celery.task.control import inspect, revoke
from django.contrib.auth import get_user_model
from django.db import models

from reservations.tasks import check_basket, execute_reservation_job

User = get_user_model()

class TimestampedModel(models.Model):
    created_at = models.DateTimeField(auto_now_add=True)
    modified_at = models.DateTimeField(auto_now=True)

    class Meta:
        abstract = True

class ReservationJob(TimestampedModel):
    PENDING = "PENDING"
    COMPLETED = "COMPLETED"
    FAILED = "FAILED"
    CANCELLED = "CANCELLED"
    APPROVED = "APPROVED"

    STATUS_CHOICES = (
        (PENDING, "PENDING"),
        (COMPLETED, "COMPLETED"),
        (FAILED, "FAILED"),
        (CANCELLED, "CANCELLED"),
        (APPROVED, "APPROVED"),
    )

    ETA = "ETA"
    IMMEDIATE = "IMMEDIATE"

    EXECUTION_TYPE_CHOICES = ((ETA, "ETA"), (IMMEDIATE, "IMMEDIATE"))

    execution_time = models.DateTimeField()
    execution_type = models.CharField(
        max_length=100,
        null=True,
        blank=True,
        choices=EXECUTION_TYPE_CHOICES,
        default=ETA,
    )
    status = models.CharField(
        max_length=255, null=True, blank=True, choices=STATUS_CHOICES, default=PENDING
    )
    selection = models.ForeignKey("selections.Selection", on_delete=models.CASCADE)
    user = models.ForeignKey(
        User,
        related_name="reservation_jobs",
        on_delete=models.CASCADE,
        null=True,
        blank=True,
    )

    def __str__(self):
        return f"<ReservationJob> {self.execution_time.strftime('%Y-%m-%d %H:%M')} {self.status}"

    class Meta:
        ordering = ("selection__slot__date_time",)

    def revoke(self):
        i = inspect()
        scheduled_tasks = i.scheduled()
        for _, tasks in scheduled_tasks.items():
            rj_task = [task for task in tasks if task["request"]["args"][0] == self.id]
            if rj_task:
                revoke(rj_task[0]["request"]["id"])

    def execute(self):
        if self.status == self.CANCELLED:
            return

        if self.execution_type == self.IMMEDIATE:
            execute_reservation_job.delay(self.id)
        else:
            execute_reservation_job.apply_async((self.id,), eta=self.execution_time)

    def save(
        self, force_insert=False, force_update=False, using=None, update_fields=None
    ):
        if self.status == self.CANCELLED:
            self.revoke()

        super().save(force_insert, force_update, using, update_fields)
        self.execute()

class Reservation(TimestampedModel):
    PENDING = "PENDING"
    IN_CART = "IN_CART"
    FAILED = "FAILED"
    REMOVED_FROM_BASKET = "REMOVED_FROM_BASKET"
    PAID = "PAID"
    EXPIRED = "EXPIRED"

    STATUS_CHOICES = (
        (IN_CART, "IN_CART"),
        (FAILED, "FAILED"),
        (PENDING, "PENDING"),
        (REMOVED_FROM_BASKET, "REMOVED_FROM_BASKET"),
        (PAID, "PAID"),
        (EXPIRED, "EXPIRED"),
    )

    user = models.ForeignKey(
        User, related_name="reservations", on_delete=models.CASCADE
    )
    selection = models.ForeignKey("selections.Selection", on_delete=models.CASCADE)
    status = models.CharField(
        max_length=255, null=True, blank=True, choices=STATUS_CHOICES, default=PENDING
    )

    def __str__(self):
        return f"<Reservation> {self.user} {self.status} {self.selection}"

    class Meta:
        ordering = ("selection__slot__date_time",)

    @property
    def is_success(self):
        return self.status == Reservation.IN_CART

    def save(
        self, force_insert=False, force_update=False, using=None, update_fields=None
    ):
        from reservations.helpers import send_reservation_email

        super().save(force_insert, force_update, using, update_fields)

        if self.status in (Reservation.IN_CART, Reservation.FAILED):
            send_reservation_email(self)

            status = (
                ReservationJob.COMPLETED if self.is_success else ReservationJob.FAILED
            )
            ReservationJob.objects.filter(
                user=self.user, selection=self.selection
            ).update(status=status)

        if self.status == Reservation.IN_CART:
            check_basket.apply_async(
                (self.id,), eta=self.created_at + timedelta(hours=12)
            )


'''
Given a collection of distinct integers, return all possible permutations.

Example:

Input: [1,2,3]
Output:
[
  [1,2,3],
  [1,3,2],
  [2,1,3],
  [2,3,1],
  [3,1,2],
  [3,2,1]
]

concept:

test cases:

recursive vs iterative:

'''
import math
import itertools
class Solution(object):
    def permute(self, nums):
        """
        :type nums: List[int]
        :rtype: List[List[int]]
        """
        return list(itertools.permutations(nums))

''' without itertools, recursive, loops'''
def dfs(nums, array, answer):
    if len(array) == len(nums):
        #print(array)
        answer.append(array)
        if len(answer) == 2:
            print(answer)
            return answer
    else:
        for i in range(0,len(nums)):
            if nums[i] in array:
                continue
            array.append(nums[i])
            dfs(nums, array, answer)
            array.pop()

a = []
b = dfs([1,2],[],a)
print(a)
print(b)


from PIL import Image
import numpy as np

import matplotlib.pyplot as plt
from thresholding import threshold

i = Image.open('images/numbers/0.1.png')
iar = np.array(i)
i2 = Image.open('images/numbers/y0.4.png')
iar2 = np.array(i2)
i3 = Image.open('images/numbers/y0.5.png')
iar3 = np.array(i3)
i4 = Image.open('images/facebook.png')
iar4 = np.array(i4)

iar = threshold(iar)
iar2 = threshold(iar2)
iar3 = threshold(iar3)
iar4 = threshold(iar4)

fig = plt.figure()
ax1 = plt.subplot2grid((8,6),(0,0), rowspan=4, colspan=3)
ax2 = plt.subplot2grid((8,6),(4,0), rowspan=4, colspan=3)
ax3 = plt.subplot2grid((8,6),(0,3), rowspan=4, colspan=3)
ax4 = plt.subplot2grid((8,6),(4,3), rowspan=4, colspan=3)

ax1.imshow(iar)
ax2.imshow(iar2)
ax3.imshow(iar3)
ax4.imshow(iar4)

plt.show()

Labmaka Internal Training/P2: Image Recognition/testreshold.py


#coding=utf-8
import socketserver
import json
from lib.utils.HttpWappalyzer import HttpWappalyzer
from conf.ConfigFileModifyHandler import *
import copy
import queue
import ast
from socketserver import UDPServer
from socketserver import ThreadingMixIn
from socketserver import BaseRequestHandler
from lib.plugins.ssrf import SSRF
from lib.plugins.rce import RCE
from lib.plugins.sql_error import SQLError
from lib.plugins.sql_bool import SQLBool
from lib.utils.diffpage_score import Diffpage_score
#from lib.utils.CosineSimilarity import CosineSimilarity
from conf.ConfigFileModifyHandler import Config
import hashlib
import threading

class MyUDPServer(ThreadingMixIn, UDPServer):
    def __init__(self, server_address, RequestHandlerClass, bind_and_activate=True, queue=None):
        self.queue = queue
        UDPServer.__init__(self, server_address, RequestHandlerClass, bind_and_activate=bind_and_activate)

class MyUDPHandler(socketserver.BaseRequestHandler):
    def __init__(self, request, client_address, server):
        self.queue = server.queue
        BaseRequestHandler.__init__(self, request, client_address, server)

    def parse(self,p):
        x = {}
        for k,v in p.items():
            try:
                v1 = json.loads(v)
            except:
                v1 = v
            x[k] = v1
        return x

    def handle(self):  # 必须要有handle方法；所有处理必须通过handle方法实现
        # self.request is the TCP socket connected to the client
        self.data = self.request[0].strip()
        data_dict = ast.literal_eval(self.data.decode('utf-8'))
        data_dict['param_in_url'] = self.parse(data_dict['param_in_url'])
        data_dict['param_in_body'] = self.parse(data_dict['param_in_body'])
        self.queue.put(data_dict)

class DeDuplicate(object):
    def __init__(self,duplicate_list,logger):
        self.duplicate_list = duplicate_list
        self.logger = logger
        self.duplicate_params = Config.get_instance().get("app.DUPLICATE_PARAMS").split('|')
        self.duplicate_length = int(Config.get_instance().get('app.DUPLICATE_LEGNTH'))
        self.black_host_list =  Config.get_instance().get("app.BLACK_HOSTS").split('|')

    def getHash(self,hashString):
        m = hashlib.md5(hashString.encode(encoding='utf-8'))
        return m.hexdigest()  # 转化为16进制打印md5值

    def pop_param(self, param_list):
        """
        去掉黑名单的headers
        :param headers_dict:
        :return:
        """
        keys = param_list.keys()
        for key in list(keys):
            if key in self.duplicate_params:
                param_list.pop(key)

        return param_list

    def query(self,request,http):
        host = request['host']
        #print(host)
        #print(self.black_host_list)
        if host in self.black_host_list:
            self.logger.info('黑名单host, pass')
            return False
        uri = http.parseUrl(request['full_url'])
        content_type = request['content_type']
        param_in_body = request['param_in_body']
        param_in_url = request['param_in_url']
        try:
            #self.logger.info(request['body'])
            body = json.loads(request['body'])
        except:
            body = {}
        tmpObj = [param_in_body, param_in_url,body]
        listobj = list(map(self.pop_param, tmpObj))
        hashString = uri + str(content_type) + ''.join([str(i) for i in listobj])
        #self.logger.info("----" + hashString)
        md5 = self.getHash(hashString)
        if md5 in self.duplicate_list:
            self.logger.info('重复请求')
            return False
        elif len(self.duplicate_list)<=self.duplicate_length:
            self.duplicate_list.append(md5)
            return True
        else:
            self.duplicate_list.clear()
            self.logger.info('exceed max length ,clear md5 list')
            self.duplicate_list.append(md5)
            return True

def worker(q):
    while True:
        logger.info(f'队列长度: {q.qsize()}')
        logger.info(f'当前运行的线程数量: {threading.active_count()}')
        data = q.get()
        scan_vul(data)

def scan_vul(data):
    lock = threading.RLock()
    deDuplicate = DeDuplicate(deplicate_list, logger)
    http = HttpWappalyzer()
    lock.acquire()
    try:
        status = deDuplicate.query(copy.deepcopy(data), http)
    except:
        status = True
    lock.release()
    if status:
        ssrf = SSRF(http)
        sql = SQLError(http)
        content_type = data['content_type']
        sqlbool = SQLBool(http, model, content_type)

        try:
            ssrf.scan(copy.deepcopy(data))
        except Exception as e:
            logger.error(str(e))
            pass
        try:
            sql.scan(copy.deepcopy(data))
        except Exception as e:
            logger.error(str(e))
            pass
        try:
            sqlbool.scan(copy.deepcopy(data))
        except Exception as e:
            logger.error(str(e))
            pass

if __name__ == "__main__":
    logger = CommonLog(__name__).getlog()
    HOST, PORT = "0.0.0.0", 32743
    queue = queue.Queue()
    #model = CosineSimilarity()
    model = Diffpage_score()
    server = MyUDPServer((HOST, PORT), MyUDPHandler, queue=queue)  # 实例化一个多线程UDPServer
    server.max_packet_size = 8192 * 20
    # Start the server
    SERVER_THREAD = threading.Thread(target=server.serve_forever)
    SERVER_THREAD.daemon = True
    SERVER_THREAD.start()

    #http = HttpWappalyzer()
    deplicate_list = []
    #deDuplicate = DeDuplicate(deplicate_list, logger)

    logger.info(f'----- udp server start at {HOST} port {PORT} ----')
    thread_num = 10
    threads = []
    for i in range(thread_num):
        t = threading.Thread(target=worker, args=(queue,))
        threads.append(t)

    for i in range(thread_num):
        threads[i].start()

    for i in range(thread_num):
        threads[i].join()


from tkinter import W
import numpy as np
from scipy.spatial import distance

class KNN:
	def __init__(self, n):
		self.k = n

	def prepare(self, X, Y):
		# find all the distances of all the data nxn matrix of distances
		self.X = X
		self.Y = Y

		self.distances = np.zeros((Y.shape[0], Y.shape[0]))

		# calculate the distances for later
		for i in range(self.distances.shape[0]):
			for j in range(self.distances.shape[1]):
				if(i == j):
					self.distances[i][j] = 0
				else:
					self.distances[i][j] = self.distances[j][i] = distance.euclidean(X[i], X[j])

		return

	def find_label(self, i):
		# after fitting this can predict the label for a dp
		row = self.distances[i].tolist()
		labels = self.Y.tolist()

		knn_labels = []
		visited = []

		for i in range(self.k):
			min = float("inf")
			best_idx = -1
			for idx, distance in enumerate(row):
				if(idx not in visited and  distance < min and distance != 0):
					min = distance
					best_idx = idx

			visited.append(best_idx)
			knn_labels.append(labels[best_idx])

		counts = {}
		for l in knn_labels:
			if(str(l) in counts):
				counts[str(l)] += 1
			else:
				counts[str(l)] = 1

		label = ""
		count = 0
		for l in counts:
			if(counts[l] > count):
				label = l

		return label

	def predict(self, dp):
		return

		return counts


import locale

locale.setlocale(locale.LC_NUMERIC, 'C')

import numpy as np
from numba import njit, jit, jitclass, float64, int64, void, boolean, uint64, complex128, prange
import cmath

#c = 2.998e8  # m/s
c = 1.0  # m/s

modes = {
    "spherical": 1,
    "gaussian": 2,
    "ray": 3,
}

@njit
def unit_vector(vector):
    """ Returns the unit vector of the vector.  """
    return vector / np.linalg.norm(vector)

@njit
def rotate_vector(vector, theta):
    c, s = np.cos(theta), np.sin(theta)
    R = np.zeros((2, 2))
    R[0, 0] = c
    R[1, 0] = -s
    R[0, 1] = s
    R[1, 1] = c
    #return np.matmul(R, vector)
    return R @ vector

# @jit(nopython=True)
# def gen_rotation_matrix(theta):
#     c, s = np.cos(theta), np.sin(theta)
#     R = np.array([[c, -s], [s, c]])
#     return R

@njit
def weightedChoice(weights):
    """
    Return a random item from objects, with the weighting defined by weights
    (which must sum to 1).
    From: http://stackoverflow.com/a/10803136
    """
    cs = np.cumsum(weights)  # An array of the weights, cumulatively summed.
    idx = np.sum(cs < np.random.rand())  # Find the index of the first weight over a random value.
    return idx

@njit
def determinant(v, w):
    return v[0] * w[1] - v[1] * w[0]

@jit('double(double[:], double[:])',nopython=True)
def angle_between(v1, v2):
    """ Returns the angle in radians between vectors 'v1' and 'v2'::
    """
    v1_u = unit_vector(v1)
    v2_u = unit_vector(v2)
    v = np.dot(v1_u, v2_u)

    if determinant(v1,v2) > 0:
        return np.real(cmath.acos(v))
    else:
        return np.real(-cmath.acos(v))

# calculation of line intersection from
# https://stackoverflow.com/questions/20677795/how-do-i-compute-the-intersection-point-of-two-lines-in-python
@njit
def line(p1, p2):
    A = (p1[1] - p2[1])
    B = (p2[0] - p1[0])
    C = (p1[0] * p2[1] - p2[0] * p1[1])
    return A, B, -C

@njit
def intersection(L1, L2):
    D = L1[0] * L2[1] - L1[1] * L2[0]
    Dx = L1[2] * L2[1] - L1[1] * L2[2]
    Dy = L1[0] * L2[2] - L1[2] * L2[0]
    x = Dx / D
    y = Dy / D
    return x, y

@njit
def get_intersect(A, B, C, D):
    a1 = B[1] - A[1]
    b1 = A[0] - B[0]
    c1 = a1 * (A[0]) + b1 * (A[1])

    a2 = D[1] - C[1]
    b2 = C[0] - D[0]
    c2 = a2 * (C[0]) + b2 * (C[1])

    determinant = a1 * b2 - a2 * b1

    if determinant == 0:
        return np.inf, np.inf
    else :
        x = (b2 * c1 - b1 * c2) / determinant
        y = (a1 * c2 - a2 * c1) / determinant
        return x,y

spec_Wavelets = [
    ('r', float64[:,:]),
    ('k', float64[:,:]),
    ('t0', float64[:]),
    ('phases', float64[:]),
    ('intensities', float64[:]),
    ('surface_index', int64[:]),
    ('wavelength', float64),
    #('f', float64),
    ('mode', uint64),
    ('n', int64)
]

@jitclass(spec_Wavelets)
class Wavelets(object):
    def __init__(self, r, k, t0, wavelength, phases, mode):
        self.r = r
        self.n = r.shape[0]
        self.k = np.zeros((self.n,2))
        for i in range(self.n):
            self.k[i,:] = ((k[i,:] / np.linalg.norm(k[i,:])) * 2 * np.pi) / wavelength
        self.t0 = t0
        self.phases = phases
        self.wavelength = wavelength
        #self.f = c / wavelength
        self.mode = mode
        self.intensity = np.ones(self.n)
        self.surface_index = np.zeros(self.n,dtype=np.int64)

#@njit(void(ExposurePoints.class_type.instance_type, ContourPoints.class_type.instance_type), parallel=False)
#def calc_forces(epoints, cpoints):

    def calc_optical_path_of_wavelet(self, index, point, n=1.0):
        f = (c/n) / self.wavelength
        #return np.sqrt(np.sum(np.square(np.subtract(self.r[index,:], point)))) / (c/n) + self.t0[index]
        #self.phases[index] += np.dot(np.subtract(self.r[index,:], point),self.k[index,:])*n
        self.phases[index] += 2 * cmath.pi * f *np.sqrt(np.sum(np.square(np.subtract(self.r[index,:], point)))) / (c/n)

    def calc_field(self, points, t, n=1.0):
        f = (c/n) / self.wavelength
        field = np.zeros(points.shape[0],dtype=np.float64)
        for j in range(points.shape[0]):
            for i in range(self.n):
                r = self.r[i,:] - points[j,:]
                field[j] += 1 / r * cmath.exp(1j * ( ( np.dot(self.k[i, :],r)
                                                     - 2 * cmath.pi * f * (t - self.t0[i]) + self.phases[i])))
        return np.real(field)

    def field_at_r(self,t,index):
        n = 1.0
        f = (c/n) / self.wavelength
        field = np.real(cmath.exp(1j *( -2 * cmath.pi * f * (t - self.t0[index]) + self.phases[index])))
        return field#*rotate_vector(self.k[index,:],np.pi/2) / np.linalg.norm(self.k[index,:])

    def phasor_at_r(self,index):
        n = 1.0
        f = (c/n) / self.wavelength
        phase = 2 * cmath.pi * f * self.t0[index] + self.phases[index]
        #phase = phase % (np.pi * 2)
        return  cmath.exp(1j*phase)#.real, cmath.exp(1j*phase).imag  #*rotate_vector(self.k[index,:],np.pi/2) / np.linalg.norm(self.k[index,:])

    def calc_probability_of_wavelet(self,index, point1, point2):
        v1 = np.subtract(point1,self.r[index, :])
        v2 = np.subtract(point2,self.r[index, :])

        if self.mode == 1: # spherical
            phi2 = angle_between(v1, v2)
            probability = np.abs((phi2) / (np.pi))

        elif self.mode == 2: # gaussian
            phi1 = angle_between(v1, self.k[index, :])
            phi2 = angle_between(v2, self.k[index, :])
            if phi2 > phi1:
                probability = np.real( 1 / (4 * np.pi) * (
                            cmath.sin(2 * phi1 + np.pi) - 2 * phi1 - cmath.sin(2 * phi2 + np.pi) + 2 * phi2))
            else:
                probability = np.real( 1 / (4 * np.pi) * (
                            cmath.sin(2 * phi2 + np.pi) - 2 * phi2 - cmath.sin(2 * phi1 + np.pi) + 2 * phi1))
            # if phi1 == phi2:
            #     probability = np.real( 1 / (8 * np.pi))

        elif self.mode == 3: #ray
            if self.is_between(self.k[index, :],v1,v2):
                probability = 1.0
            else:
                probability = 0.0

        return probability

    def angle_between_clockwise(self, v1, v2):
        """ Returns the angle in radians between vectors 'v1' and 'v2'::
        """
        def determinant(v, w):
            return v[0] * w[1] - v[1] * w[0]
        v1_u = v1 / np.linalg.norm(v1)
        v2_u = v2 / np.linalg.norm(v2)
        v = np.dot(v1_u, v2_u)
        if determinant(v1,v2) > 0:
            return cmath.acos(v)
        else:
            return 2*np.pi-cmath.acos(v)

    def angle_between_anticlockwise(self, v1, v2):
        """ Returns the angle in radians between vectors 'v1' and 'v2'::
        """
        def determinant(v, w):
            return v[0] * w[1] - v[1] * w[0]
        v1_u = v1 / np.linalg.norm(v1)
        v2_u = v2 / np.linalg.norm(v2)
        v = np.dot(v1_u, v2_u)
        if determinant(v1,v2) < 0:
            return cmath.acos(v)
        else:
            return 2*np.pi-cmath.acos(v)

    def is_between(self,k,v1,v2):
        def determinant(v, w):
            return v[0] * w[1] - v[1] * w[0]
        return (determinant(k,v1)>0) ^ (determinant(k,v2)>0)

    def append_wavelets(self, wavelets):
        self.r = np.concatenate((self.r, wavelets.r))
        self.k = np.concatenate((self.k, wavelets.k))
        self.n += wavelets.n
        self.t0 = np.concatenate((self.t0, wavelets.t0))
        self.phases = np.concatenate((self.phases, wavelets.phases))

spec_Surface = [
    ('points', float64[:, :]),
    ('reflectivity', float64),
    ('transmittance', float64),
    ('n1', float64),
    ('n2', float64),
    ('midpoints', float64[:, :]),
    ('field', float64[:,:]),
    ('phasor', complex128[:]),
    ('hits', int64[:]),
    ('normals', float64[:, :]),
    ('count', int64),
]

@jitclass(spec_Surface)
class Surface(object):
    def __init__(self, points, reflectivity, transmittance, n1=1.0, n2=1.0):
        self.points = points
        self.reflectivity = reflectivity
        self.transmittance = transmittance
        self.n1 = n1
        self.n2 = n2
        self.midpoints = np.zeros((self.points.shape[0]-1,2))
        self.normals = np.zeros((self.points.shape[0] - 1,2))
        for i in range(self.points.shape[0]-1):
            self.midpoints[i] = 0.5 * np.add(self.points[i], self.points[i + 1])
            normal = rotate_vector(np.subtract(self.points[i], self.points[i + 1]), np.pi / 2)
            normal = unit_vector(normal)
            self.normals[i] = normal

        self.field = np.zeros((self.midpoints.shape[0],2),np.float64)
        self.phasor = np.zeros(self.midpoints.shape[0],dtype=np.complex128)
        self.hits = np.zeros(self.midpoints.shape[0],dtype=np.int64)
        self.count = 0

    def _update_midpoints(self):
        for i in range(self.points.shape[0]-1):
            self.midpoints[i] = 0.5 * np.add(self.points[i], self.points[i + 1])

    def _update_normals(self):
        for i in range(self.points.shape[0]-1):
            normal = rotate_vector(np.subtract(self.points[i], self.points[i + 1]), np.pi / 2)
            normal = unit_vector(normal)
            self.normals[i] = normal

    def flip_normals(self):
        for i in range(self.normals.shape[0]):
            self.normals[i] =  rotate_vector(self.normals[i], np.pi )

    def reflected_k(self, vector, normal):
        reflected = vector - (2 * (np.dot(normal, vector)) * normal)
        return reflected

    def transmitted_k(self, k, normal):
        alpha = angle_between(k, normal)
        beta = cmath.asin( (self.n1/self.n2) * np.sin(alpha) ).real
        transmitted = rotate_vector(normal, beta)
        return transmitted

    def localize_wavelet(self, wavelets, index):
        probabilities = np.zeros(self.points.shape[0] - 1, dtype=np.float64)
        for i in range(self.points.shape[0] - 1):
            probabilities[i] = wavelets.calc_probability_of_wavelet(index,self.points[i], self.points[i + 1])

        if (wavelets.mode == 1) or (wavelets.mode == 2):
            # if np.random.rand() >= np.sum(probabilities):
            #     probabilities /= np.sum(probabilities)
            #     index = self.weightedChoice(probabilities)
            #     return self.midpoints[index], self.normals[index], True
            # else:
            #     return np.array([0.0, 0.0]), np.array([0.0, 0.0]), False
            probabilities /= np.sum(probabilities)
            surface_index = weightedChoice(probabilities)
            return surface_index, self.midpoints[surface_index,:],True

        elif wavelets.mode == 3:
            for j in range(len(probabilities)):
                if probabilities[j] > 0:
                    #x,y = intersection(line(wavelets.r[index,:],wavelets.r[index,:]+wavelets.k[index,:]),line(self.points[i],self.points[i+1]))
                    x,y = get_intersect(wavelets.r[index,:],wavelets.r[index,:]+wavelets.k[index,:],self.points[j],self.points[j+1])
                    return j, np.array([x,y]) ,True
            return -1, np.array([0.0,0.0]), False

    def interact_with_wavelet(self, wavelets, index):
        surface_index, point, hit = self.localize_wavelet(wavelets, index)
        absorbed = True
        if hit:
            if self.reflectivity > 0.0:
                if np.random.rand() <= self.reflectivity:
                    new_k = self.reflected_k(wavelets.k[index,:], self.normals[surface_index,:])
                    absorbed = False
            elif self.transmittance > 0.0:
                if np.random.rand() <= self.transmittance:
                    new_k = self.transmitted_k(wavelets.k[index,:], self.normals[surface_index,:])
                    absorbed = False

            if not absorbed:
                #wavelets.calc_optical_path_of_wavelet(index, self.midpoints[surface_index, :], self.n1)
                wavelets.calc_optical_path_of_wavelet(index, point, self.n1)
                #k = k / np.linalg.norm(k) * 2 * np.pi / wavelets.wavelength
                return surface_index, point, new_k, True

        return -1, np.array([0.0, 0.0]),np.array([0.0, 0.0]), False

    def add_field_from_wavelets(self,wavelets):
        self.count += wavelets.n

        for i in range(wavelets.n):
            j = wavelets.surface_index[i]
            self.field[j] += wavelets.field_at_r(i,1.0)#*np.sin(np.real(angle_between(self.rotate_vector(wavelets.k[i,:],np.pi/2),(self.points[j+1, :]-self.points[j, :]))))
            #self.field[j] += np.cos((wavelets.phase_at_r(i)-self.phase[j])/2)
            self.hits[j] += 1

    def add_phase_from_wavelets(self,wavelets):
        self.count += wavelets.n

        for i in range(wavelets.n):
            j = wavelets.surface_index[i]
            #self.phase[j] += ( (wavelets.phase_at_r(i)  ) + self.phase[j] )/2
            self.phasor[j] += wavelets.phasor_at_r(i)
            self.hits[j] += 1

    def interact_with_all_wavelets(self, wavelets):
        hits = np.zeros(wavelets.n, dtype=np.float64)
        rs = np.zeros((wavelets.n,2), dtype=np.float64)
        ks = np.zeros((wavelets.n,2), dtype=np.float64)
        ts = np.zeros(wavelets.n, dtype=np.float64)
        s_index = np.zeros(wavelets.n,dtype=np.int64)

        r = np.zeros(2)
        k = np.zeros(2)
        t = 0.0
        hit = False

        for i in range(wavelets.n):
            surface_index, point, k, hit = self.interact_with_wavelet(wavelets,i)
            hits[i] = hit
            rs[i] = point
            ks[i] = k
            ts[i] = t
            s_index[i] = surface_index

        indices = (hits > 0)
        new_wavelets = Wavelets(rs[indices,:],ks[indices,:],ts[indices],wavelets.wavelength,wavelets.phases[indices],wavelets.mode)
        new_wavelets.surface_index = s_index[indices]
        return new_wavelets

    def clear(self):
        self.field = np.zeros((self.midpoints.shape[0], 2), np.float64)
        self.hits = np.zeros(self.midpoints.shape[0], dtype=np.int64)

class Lense(object):

    def __init__(self, x, y, r1, r2, height,reflectivity=0.0,transmittance=1.0, n1=1.0, n2=1.5, num=128):
        self.n1 = n1
        self.n2 = n2
        self.r1 = r1
        self.r2 = r2
        self.x = x
        self.y = y
        self.height = height

        points1 = self._generate_lens_points(self.r1,self.height,num)

        points2 = self._generate_lens_points(self.r2,self.height,num)

        self.front = Surface(points1, reflectivity=reflectivity, transmittance=transmittance, n1=n1, n2=n2)
        self.back = Surface(points2, reflectivity=reflectivity, transmittance=transmittance, n1=n2, n2=n1)

        self.front.points[:,0] -= self.height / 5
        self.back.points[:, 0] += self.height / 5

        self.front.points[:,0] += x
        self.back.points[:, 0] += x
        self.front.points[:,1] += y
        self.back.points[:, 1] += y

        self.front._update_midpoints()
        self.front._update_normals()
        self.back._update_midpoints()
        self.back._update_normals()

        if self.r1 is not np.inf:
            self.front.flip_normals()
        if self.r2 is not np.inf:
            self.back.flip_normals()

        #self.front.flip_normals()
        self.d = self.back.points[:, 0].max() - self.front.points[:, 0].min()

        self.f = self._calc_f()
        print('d: '+str(self.d))
        print('f: ' + str(self.f))

    def _gen_concave_points(self, p0, r, height, num):
        if height > np.abs(r):
            height = np.abs(r)
        v = np.array([r, 0])
        theta_max = np.arcsin(height / r)
        thetas = np.linspace(theta_max, -theta_max, num)
        points = np.zeros((num, 2))
        for i, theta in enumerate(thetas):
            # R = gen_rotation_matrix(theta)
            # buf = np.dot(R, v)
            buf = rotate_vector(v,theta)
            points[i] = p0 + buf
        return points

    def _generate_lens_points(self,r,height, num):

        if (r is not np.inf) and (r is not -np.inf):
            p0 = np.array([0.0, 0.0])
            points1 = self._gen_concave_points(p0, r, height, num)
            if r > 0:
                points1[:, 0] -= points1[:, 0].min()
            else:
                points1[:, 0] -= points1[:, 0].max()
        else:
            points1 = np.zeros((num, 2))
            points1[:, 1] = np.linspace(self.height, -self.height, num)

        return points1

    def _calc_f(self):
        return np.abs(1/ ( (self.n2-self.n1)/self.n1 * ( 1 / self.r1 - 1/ self.r2) + ((self.n2-self.n1)**2*self.d)/(self.n2*self.n1*self.r1*self.r2) ))

    def _calc_f_back(self):
        h1 = -(self.f*(self.n2-1)*self.d)/( self.r2*self.n2)
        return np.abs((self.f-h1) - (-self.front.points[:,0].min()+self.x))

    def _calc_f_front(self):
        h2 = -(self.f*(self.n2-1)*self.d)/( self.r1*self.n2)
        return np.abs((self.f-h2) - (self.back.points[:,0].max()-self.x))

    def shift(self, dx = 0, dy = 0):
        self.x += dx
        self.y += dy
        self.front.points[:, 0] += dx
        self.back.points[:, 0] += dx
        self.front.points[:, 1] += dy
        self.back.points[:, 1] += dy
        self.front._update_midpoints()
        self.back._update_midpoints()

class HyperbolicLense(object):

    def __init__(self, x, y, f, height,reflectivity=0.0,transmittance=1.0, n1=1.0, n2=1.5, num=128):
        self.n1 = n1
        self.n2 = n2
        self.f = f
        self.x = x
        self.y = y
        self.height = height
        self.reflectivity = reflectivity
        self.transmittance = transmittance
        self.num = num

        self._make_surfaces()

    def _make_surfaces(self,flipped = False):

        if flipped:
            points2 = self._gen_hyperbolic_points()
            for i in range(self.front.points.shape[0]):
                points2[i, :] = rotate_vector(self.front.points[i,:], np.pi)
            points2 = np.flipud(points2)
            points1 = self._generate_line()
        else:
            points1 = self._gen_hyperbolic_points()
            points2 = self._generate_line()

        self.front = Surface(points1, reflectivity=self.reflectivity, transmittance=self.transmittance, n1=self.n1, n2=self.n2)
        self.back = Surface(points2, reflectivity=self.reflectivity, transmittance=self.transmittance, n1=self.n2, n2=self.n1)

        self.front.points[:,0] -= self.height / 5
        self.back.points[:, 0] += self.height / 5

        self.front.points[:,0] += self.x
        self.back.points[:, 0] += self.x
        self.front.points[:,1] += self.y
        self.back.points[:, 1] += self.y

        self.front._update_midpoints()
        self.front._update_normals()
        self.back._update_midpoints()
        self.back._update_normals()

        #self.front.flip_normals()
        self.d = self.back.points[:, 0].max() - self.front.points[:, 0].min()

    # from :
    # https://www.google.de/url?sa=t&rct=j&q=&esrc=s&source=web&cd=6&cad=rja&uact=8&ved=2ahUKEwij4-Ly6M7cAhVRKuwKHW6QDCcQFjAFegQICRAC&url=https%3A%2F%2Fwww.ssl.berkeley.edu%2F~mlampton%2FHyperbolicLens.pdf&usg=AOvVaw2TZIPNb1xqk1ap_BvLLGZb
    def _gen_hyperbolic_points(self):
        C = 1/((self.n2-1)*self.f)
        S = 1-self.n2**2

        y = np.linspace(self.height, -self.height, self.num)
        x = np.zeros(self.num)
        x = C*(y**2) / (1 + np.sqrt( 1 - S*(C**2)*(y**2)) )

        x = x - x.max()
        points = np.zeros((self.num, 2))
        points[:, 0] = x
        points[:, 1] = y
        return points

    def _generate_line(self):

        points1 = np.zeros((self.num, 2))
        points1[:, 1] = np.linspace(self.height, -self.height, self.num)

        return points1

    def shift(self, dx = 0, dy = 0):
        self.x += dx
        self.y += dy
        self.front.points[:, 0] += dx
        self.back.points[:, 0] += dx
        self.front.points[:, 1] += dy
        self.back.points[:, 1] += dy
        self.front._update_midpoints()
        self.back._update_midpoints()

    # def fliplr(self):
    #     self.front.points[:, 0] -= self.x
    #     self.back.points[:, 0] -= self.x
    #     self.front.points[:, 1] -= self.y
    #     self.back.points[:, 1] -= self.y
    #     for i in range(self.front.points.shape[0]):
    #         self.front.points[i, :] = rotate_vector(self.front.points[i,:], np.pi)
    #         self.back.points[i, :] = rotate_vector(self.back.points[i,:], np.pi)
    #     self.front.points[:, 0] += self.x
    #     self.back.points[:, 0] += self.x
    #     self.front.points[:, 1] += self.y
    #     self.back.points[:, 1] += self.y
    #     self.front._update_midpoints()
    #     self.front._update_normals()
    #     #self.front.flip_normals()
    #     self.back._update_midpoints()
    #     self.front._update_normals()
    #     buf = self.front
    #     self.front = self.back
    #     self.back = buf
    #     self.back.flip_normals()
    #     self.back.n1 = self.n2
    #     self.back.n2 = self.n1
    #     self.front.n1 = self.n1
    #     self.front.n2 = self.n2

@jit
def make_dipole(wavelength, theta,alpha_max,num, mode='ray'):
    rs = np.zeros((num, 2))
    ks = np.zeros((num, 2))
    ks[:, 0] = np.repeat(1.0, num)
    alphas = np.linspace(-alpha_max, alpha_max, num)
    probabilities = (np.cos(theta - alphas) ** 2)
    #probabilities = np.ones(num)
    probabilities /= np.sum(probabilities)
    b = np.zeros(num)

    for i in range(ks.shape[0]):
        index = weightedChoice(probabilities)
        b[i] = alphas[index]
        ks[i, :] = rotate_vector(ks[i, :], alphas[index])

    phases = np.zeros((num))
    for i in range(len(alphas)):
        if theta-b[i]-np.pi/2 < 0:
            phases[i] = np.pi

    t0s = np.zeros((num))
    return Wavelets(r=rs, k=ks, t0=t0s, wavelength=wavelength, phases=phases, mode=modes[mode])


from django.urls import path
from django.contrib.auth import views as auth_views
# from django.contrib.auth import views
from . import views

app_name = 'blog'
urlpatterns = [
    # path("articles_check/", views.articles_display, name="articles_display"),

    path('', views.homepage_view, name='homepage_view'),
    path('blogs/', views.blog_index, name="blog_index"),
    path('search_result/', views.blog_index_search, name="blog_index_search"),
    path('search_results/', views.blog_index_searched, name="blog_index_searched"),
    path('popular/', views.popular_views, name="popular_views"),
    path("new/", views.blog, name="blog"),
    path("contact-us/", views.blog_contact, name="blog_contact"),
    path("<slug:slug>/", views.blog_detail, name="blog_detail"),
    path("legal/<slug:slug>/", views.blog_detail_legal, name="blog_detail_legal"),
    path("pending/<slug:slug>/", views.pending_blog_detail, name="pending_blog_detail"),
    path("articles_list/<category>/", views.blog_category, name="blog_category"),
    path("tags_list/<tag>/", views.blog_tag, name="blog_tag"),
    path("dashboard/authors/", views.dashboard_authors, name="dashboard_authors"),
    path("dashboard/pending/", views.dashboard_pending_post, name="dashboard_pending_post"),
    path("author/<author>/", views.blog_author, name="blog_author"),
    path("dashboard/<author>/", views.dashboard_home, name="dashboard_home"),
    path("dashboard/<author>/messaging", views.dashboard_messaging, name="dashboard_messaging"),
    path("dashboard/<author>/message/<uidb64>/", views.dashboard_opened_message, name="dashboard_opened_message"),
    path("dashboard/<author>/request", views.dashboard_request, name="dashboard_request"),
    path("dashboard/<author>/collaburation", views.dashboard_collaburation, name="dashboard_collaburation"),
    path("dashboard/<author>/new", views.dashboard_add_post, name="dashboard_add_post"),
    path("dashboard/<author>/posts", views.dashboard_view_post, name="dashboard_view_post"),
    path("dashboard/<author>/edit", views.dashboard_view_post_edit, name="dashboard_view_post_edit"),
    path("dashboard/<author>/posts-collaburation", views.dashboard_collabutation_view, name="dashboard_collabutation_view"),
    path("dashboard/<author>/profile", views.dashboard_profile, name="dashboard_profile"),
    path("dashboard/<slug:slug>/edit/", views.edit_blog, name="edit_blog"),
    path("dashboard/collaburation/<slug:slug>/edit/", views.collaburation_edit_blog, name="collaburation_edit_blog"),
    path("<slug:slug>/delete/", views.delete_blog, name='delete_blog'),
    path("ajax/addnewsletter/", views.add_newsletter_email, name='add_newsletter_email'),
    path("ajax/add_category_request/", views.add_category_request, name='add_category_request'),
    path("ajax/request_key/", views.request_key, name='request_key'),
    path("ajax/verfiy_post_collaburation/", views.verfiy_post_collaburation, name='verfiy_post_collaburation'),
    path("ajax/collaburation_handshake/", views.collaburation_handshake, name='collaburation_handshake'),
    path("ajax/request_category/", views.request_category, name='request_category'),
    path("ajax/user_send_message/", views.user_send_message, name='user_send_message'),
    # path("ajax/user_send_message_other/", views.user_send_message_other, name='user_send_message_other'),
    # path("ajax/user_send_message_admin/", views.user_send_message_admin, name='user_send_message_admin'),
    path("ajax/add_category/", views.add_category, name='add_category'),
    path("ajax/verify_post/", views.add_post_request, name='add_post_request'),
    # path("articles/<slug:slug>/", views.category_archive, name='category_archive'),
    # path("<int:pk>/", views.blog_detail, name="blog_detail"),
    # path("<int:pk>/edit/", views.edit_blog, name="edit_blog"),
    # path("<int:pk>/delete/", views.delete_blog, name='delete_blog'),
    # path("add_comment/<int:pk>/", views.add_comment, name="add_comment"),
]


import numpy as np
import pandas as pd
# from dev.dataloading.dataset import Dataset
# from dev.dataloading.domain import Domain

from utils import Dataset, Domain

CATEGORICAL = "categorical"
CONTINUOUS = "continuous"
ORDINAL = "ordinal"

class Transformer:
    """Continuous and ordinal columns are normalized to [0, 1].
    Discrete columns are converted to a one-hot vector.
    """

    def __init__(
        self, categorical_columns, continuous_columns, bin_size=None, normalize=False
    ):
        # self.target_feat_idx = target_feat_idx
        self.meta = None
        self.output_dim = None
        self.bin_size = bin_size
        self.categorical_columns = categorical_columns
        self.continuous_columns = continuous_columns
        self.normalize = normalize

    @staticmethod
    def get_metadata(
        data,
        categorical_columns=tuple(),
        continuous_columns=tuple(),
        ordinal_columns=tuple(),
        normalize=False,
    ):
        meta = []

        df = pd.DataFrame(data)
        for index in df:
            column = df[index]

            if index in categorical_columns:
                mapper = column.value_counts().index.tolist()
                meta.append(
                    {
                        "name": index,
                        "type": CATEGORICAL,
                        "size": len(mapper),
                        "i2s": mapper,
                    }
                )
            elif index in ordinal_columns:
                value_count = list(dict(column.value_counts()).items())
                value_count = sorted(value_count, key=lambda x: -x[1])
                mapper = list(map(lambda x: x[0], value_count))
                meta.append(
                    {"name": index, "type": ORDINAL, "size": len(mapper), "i2s": mapper}
                )
            elif index in continuous_columns:
                meta.append(
                    {
                        "name": index,
                        "type": CONTINUOUS,
                        "min": column.min() if normalize else 0,
                        "max": column.max() if normalize else 1,
                    }
                )

        return meta

    def fit(self, data):
        self.meta = self.get_metadata(
            data,
            self.categorical_columns,
            self.continuous_columns,
            normalize=self.normalize,
        )
        self.output_dim = 0
        for info in self.meta:
            # if info['name'] != self.target_feat_idx:
            if info["type"] in [CONTINUOUS, ORDINAL]:
                self.output_dim += 1
            else:
                self.output_dim += info["size"]

    # def transform(self, data: pd.DataFrame, target: list) -> Dataset:
    def transform(self, data: pd.DataFrame) -> Dataset:

        data_t = []
        attrs = []
        shape = []
        original_shape = []
        for id_, info in enumerate(self.meta):
            col_name = info["name"]
            col = data[col_name].to_numpy()
            attrs.append(col_name)
            if info["type"] == CONTINUOUS:
                if self.bin_size is not None:
                    col_binned = np.array(
                        pd.cut(
                            col,
                            bins=self.bin_size,
                            labels=np.arange(self.bin_size),
                            retbins=False,
                        )
                    )
                    col_binned = col_binned.reshape([-1, 1])
                    data_t.append(col_binned)
                    shape.append(self.bin_size)
                else:
                    # print(f"continuous column {id_}: {info['max']}, {info['min']}. col.max={col.max()}, col.min={col.min()}")
                    col_normed = (col - info["min"]) / (
                        info["max"] - info["min"] + 1e-9
                    )
                    data_t.append(col_normed.reshape([-1, 1]))
                    shape.append(1)
                original_shape.append(1)
            else:
                mapper = dict([(item, id) for id, item in enumerate(info["i2s"])])
                mapped = data[col_name].apply(lambda x: mapper[x]).values.astype(int)
                data_t.append(mapped.reshape([-1, 1]).astype(int))
                shape.append(info["size"])
                original_shape.append(info["size"])

        # domain = Domain(attrs, shape, target)
        domain = Domain(attrs, shape)
        df = pd.DataFrame(np.concatenate(data_t, axis=1), columns=domain.attrs)
        for id_, info in enumerate(self.meta):
            col_name = info["name"]
            if info["type"] == CATEGORICAL:
                df[col_name] = df[col_name].astype(int)

        # self.original_domain = Domain(attrs, original_shape, target)
        self.original_domain = Domain(attrs, original_shape)
        return Dataset(df, domain)

    def inverse_transform(self, data: Dataset) -> pd.DataFrame:
        """Takes as input a Dataset and return a DataFrame"""
        # data_t = np.zeros([len(data), len(self.meta)])
        data_t = pd.DataFrame()

        data = data.df.copy()
        res = []
        column_names = []
        for id_, info in enumerate(self.meta):
            col_name = info["name"]
            column_names.append(col_name)
            if info["type"] == CONTINUOUS:
                current = data.iloc[:, 0]
                data = data.iloc[:, 1:]
                min_val = info["min"]
                max_val = info["max"]
                if self.bin_size is not None:
                    # current = 0.5 * (2*current+1) / self.bin_size
                    lower = current.values / self.bin_size
                    upper = (current.values + 1) / self.bin_size
                    current = np.random.uniform(lower, upper)
                    current = current * (max_val - min_val) + min_val
                    res.append(current)
                else:
                    current = (current * (max_val - min_val) + min_val).values
                    res.append(current)
                # data_t[col_name] = current
                # print(data_t[col_name].isna().sum())
            else:
                current = data.iloc[:, 0].astype(int)
                data = data.iloc[:, 1:]
                col_data = np.array(list(map(info["i2s"].__getitem__, current)))
                res.append(col_data)
                # data_t[col_name] = list(map(info['i2s'].__getitem__, current))

        data_t = pd.DataFrame(np.column_stack(res), columns=column_names)
        for id_, info in enumerate(self.meta):
            col_name = info["name"]
            if info["type"] == CATEGORICAL:
                data_t[col_name] = data_t[col_name].astype(int)

        return data_t

if __name__ == "__main__":
    cat = ("animal",)
    con = ("age",)
    trans = Transformer(cat, con, bin_size=None)

    data = pd.DataFrame(
        [
            ["cat", 19],
            ["cat", 11],
            ["dog", 15.1],
            ["dog", 14.5],
            ["dog", 13.1],
            ["dog", 11.2],
            ["dog", 15.6],
            ["whale", 17],
            ["whale", 23],
        ],
        columns=cat + con,
    )
    trans.fit(
        data,
    )


import pytest
import logging
from azure.iot.device.iothub.models import MethodRequest, MethodResponse

logging.basicConfig(level=logging.DEBUG)

dummy_rid = 1
dummy_name = "name"
dummy_payload = {"MethodPayload": "somepayload"}
dummy_status = 200

@pytest.mark.describe("MethodRequest - Instantiation")
class TestMethodRequest(object):
    @pytest.mark.it("Instantiates with a read-only 'request_id' attribute")
    def test_request_id_property_is_read_only(self):
        m_req = MethodRequest(request_id=dummy_rid, name=dummy_name, payload=dummy_payload)
        new_rid = 2

        with pytest.raises(AttributeError):
            m_req.request_id = new_rid
        assert m_req.request_id != new_rid
        assert m_req.request_id == dummy_rid

    @pytest.mark.it("Instantiates with a read-only 'name' attribute")
    def test_name_property_is_read_only(self):
        m_req = MethodRequest(request_id=dummy_rid, name=dummy_name, payload=dummy_payload)
        new_name = "new_name"

        with pytest.raises(AttributeError):
            m_req.name = new_name
        assert m_req.name != new_name
        assert m_req.name == dummy_name

    @pytest.mark.it("Instantiates with a read-only 'payload' attribute")
    def test_payload_property_is_read_only(self):
        m_req = MethodRequest(request_id=dummy_rid, name=dummy_name, payload=dummy_payload)
        new_payload = {"NewPayload": "somenewpayload"}

        with pytest.raises(AttributeError):
            m_req.payload = new_payload
        assert m_req.payload != new_payload
        assert m_req.payload == dummy_payload

@pytest.mark.describe("MethodResponse - Instantiation")
class TestMethodResponseInstantiation(object):
    @pytest.mark.it("Instantiates with an editable 'request_id' attribute")
    def test_instantiates_with_request_id(self):
        response = MethodResponse(request_id=dummy_rid, status=dummy_status, payload=dummy_payload)
        assert response.request_id == dummy_rid

        new_rid = "2"
        assert response.request_id != new_rid
        response.request_id = new_rid
        assert response.request_id == new_rid

    @pytest.mark.it("Instantiates with an editable 'status' attribute")
    def test_instantiates_with_status(self):
        response = MethodResponse(request_id=dummy_rid, status=dummy_status, payload=dummy_payload)
        assert response.status == dummy_status

        new_status = 400
        assert response.status != new_status
        response.status = new_status
        assert response.status == new_status

    @pytest.mark.it("Instantiates with an editable 'payload' attribute")
    def test_instantiates_with_payload(self):
        response = MethodResponse(request_id=dummy_rid, status=dummy_status, payload=dummy_payload)
        assert response.payload == dummy_payload

        new_payload = {"NewPayload": "yes_this_is_new"}
        assert response.payload != new_payload
        response.payload = new_payload
        assert response.payload == new_payload

    @pytest.mark.it("Instantiates with a default 'payload' of 'None' if not provided")
    def test_instantiates_without_payload(self):
        response = MethodResponse(request_id=dummy_rid, status=dummy_status)
        assert response.request_id == dummy_rid
        assert response.status == dummy_status
        assert response.payload is None

@pytest.mark.describe("MethodResponse - .create_from_method_request()")
class TestMethodResponseCreateFromMethodRequest(object):
    @pytest.mark.it("Instantiates using a MethodRequest to provide the 'request_id'")
    def test_instantiates_from_method_request(self):
        request = MethodRequest(request_id=dummy_rid, name=dummy_name, payload=dummy_payload)
        status = 200
        payload = {"ResponsePayload": "SomeResponse"}
        response = MethodResponse.create_from_method_request(
            method_request=request, status=status, payload=payload
        )

        assert isinstance(response, MethodResponse)
        assert response.request_id == request.request_id
        assert response.status == status
        assert response.payload == payload

    @pytest.mark.it("Instantiates with a default 'payload' of 'None' if not provided")
    def test_instantiates_without_payload(self):
        request = MethodRequest(request_id=dummy_rid, name=dummy_name, payload=dummy_payload)
        status = 200
        response = MethodResponse.create_from_method_request(request, status)

        assert isinstance(response, MethodResponse)
        assert response.request_id == request.request_id
        assert response.status == status
        assert response.payload is None


import requests as res
import json
from lxml import html
z=1
pokemon_dict={}
pokemon_list=[]
def Extract_data(url):
    response=res.get(url)
    tree=html.fromstring(html=response.text)
    title=tree.xpath('/html/body/div[1]/div[2]/div/div[2]/main/div/div[2]/h1/text()')[0]
    price=('£'+str(tree.xpath('/html/body/div[1]/div[2]/div/div[2]/main/div/div[2]/p[1]/span/text()')[0]))
    stock=tree.xpath('/html/body/div[1]/div[2]/div/div[2]/main/div/div[2]/p[2]/text()')[0].split(" ")[0]
    description=tree.xpath('/html/body/div[1]/div[2]/div/div[2]/main/div/div[3]/div[1]/p/text()')[0]
    sku=tree.xpath('/html/body/div[1]/div[2]/div/div[2]/main/div/div[2]/div[2]/span[1]/span/text()')[0]
    category_count=len(tree.xpath('/html/body/div[1]/div[2]/div/div[2]/main/div/div[2]/div[2]/span[2]')[0])
    category_list=[]
    for i in range(1,category_count+1):
        xp='/html/body/div[1]/div[2]/div/div[2]/main/div/div[2]/div[2]/span[2]/a['+str(i)+']/text()'
        xp2=tree.xpath(xp)[0]
        category_list.append(xp2)
    tag_count=len(tree.xpath('/html/body/div[1]/div[2]/div/div[2]/main/div/div[2]/div[2]/span[3]')[0])
    tag_list=[]
    for j in range(1,tag_count+1):
        xp3='/html/body/div[1]/div[2]/div/div[2]/main/div/div[2]/div[2]/span[3]/a['+str(j)+']/text()'
        xp4=tree.xpath(xp3)[0]
        tag_list.append(xp4)
    pokemon={}
    pokemon_dtls={}

    pokemon['Price']=str(price)
    pokemon['Stock']=stock
    pokemon['SKU']=sku
    pokemon['Description']=str(description)
    pokemon['Categories']=category_list
    pokemon['Tags']=tag_list
    pokemon_dtls[title]=pokemon
    pokemon_list.append(pokemon_dtls)

response1=res.get('https://scrapeme.live/shop/')
tree=html.fromstring(html=response1.text)
main=tree.xpath('//*[@id="main"]/ul')[0]
itr_no=(len(main))
for i in range(1,itr_no+1):
    url='//*[@id="main"]/ul/li['+str(i)+']/a[1]'
    main2=tree.xpath(url)[0]
    Extract_data(str(html.tostring(main2)).split(" ")[1].replace("href=","").replace("\"",""))
for j in range(2,49):
    temp_url='https://scrapeme.live/shop/page/'+str(j)
    response1=res.get(temp_url)
    tree=html.fromstring(html=response1.text)
    main=tree.xpath('//*[@id="main"]/ul')[0]
    itr_no=(len(main))
    for i in range(1,itr_no+1):
        url='//*[@id="main"]/ul/li['+str(i)+']/a[1]'
        main2=tree.xpath(url)[0]
        Extract_data(str(html.tostring(main2)).split(" ")[1].replace("href=","").replace("\"",""))
    print(str(z),'/',str(775))
    z+=1
pokemon_dict['Pokemon']=pokemon_list
# print(pokemon_dict)

with open ('pokemon_1.json','w',encoding='utf-8') as write_file:
    json.dump(pokemon_dict,write_file,indent=2,ensure_ascii=False)


#E Working with Images

import matplotlib.pyplot as plt

# Loading the image
img = plt.imread('mandrill.jpg')
plt.subplot(1,2,1)
plt.imshow(img)
plt.title('Full Image')

#Cropping the image
plt.subplot(1,2,2)
img2 = img[300:900, 200:600, :]
plt.imshow(img2)
plt.title('Cropped Image')

#Saving the cropped image
plt.imsave('mandrill-face.jpg', img2)


import pandas as pd
import numpy as np
import math
import os
from Sentiment import get_sentiment_value, get_polarity_value, get_subjectivity_value
from Keyword import extract_v2, filter_text_by_pos, lemetize_text
from Utils import cleantext
import warnings
from nltk import word_tokenize
from gensim import corpora, models, similarities

warnings.simplefilter(action='ignore')

class Indicator:
    def __init__(self, data_folder):
        self.data_folder = data_folder

    @staticmethod
    def average(values_tab):
        return sum(values_tab) / len(values_tab)

    @staticmethod
    def get_mean_time_gap(tweets):
        len_t = len(tweets)
        if len_t <= 1:
            return 0

        before = tweets[0]
        cumul = 0
        for i in range(1, len_t):
            current = tweets[i]
            cumul += current - before
            before = current

        return cumul / len_t

    @staticmethod
    def nb_different_topics(texts):
        textValues = texts.values
        texts = texts.apply(cleantext)
        keywords_column = texts.apply(lambda x: extract_v2(x, withNumbers=False))
        values = keywords_column.values
        list_sets = []

        topics = []
        i = 0
        for keywords in values:
            to_remove = []
            new_kws = set(" ".join(keywords).split())

            j = 0
            for one_set in list_sets:
                inter = one_set.intersection(new_kws)
                if len(inter) > 0:
                    to_remove.append(j)
                j += 1

            if len(to_remove) > 0:
                new_set = set().union(new_kws)
                new_topic = []
                for j in to_remove:
                    one_set = list_sets[j]
                    new_set = new_set.union(one_set)
                    for topic in topics[j]:
                        new_topic.append(topic)

                new_topics = []
                new_list_sets = []
                for j in range(0, len(list_sets)):
                    if j not in to_remove:
                        new_topics.append(topics[j])
                        new_list_sets.append(list_sets[j])

                topics = new_topics
                list_sets = new_list_sets

                list_sets.append(new_set)
                topics.append(new_topic)

            elif len(new_kws) > 0:
                list_sets.append(new_kws)
                topics.append([textValues[i]])

            i += 1

        #file = open(name, 'w')
        #for topic in topics:
        #    for text in topic:
        #        file.write(text+"\n")
        #    file.write('--------------------\n')

        #file.close()
        return len(list_sets)

    @staticmethod
    def nb_different_topics_v2(name, texts):
        textValues = texts.values
        texts = texts.apply(cleantext)
        lemmetized_column = texts.apply(lambda x: lemetize_text(x))
        values = lemmetized_column.values
        list_sets = []

        topics = []

        tab_texts = [word_tokenize(text) for text in values]
        dictionary = corpora.Dictionary(tab_texts)
        feature_cnt = len(dictionary.token2id)
        corpus = [dictionary.doc2bow(text) for text in tab_texts]
        tfidf = models.TfidfModel(corpus)

        for keyword in values:
            kw_vector = dictionary.doc2bow(word_tokenize(keyword))
            index = similarities.SparseMatrixSimilarity(tfidf[corpus], num_features=feature_cnt)
            sim = index[tfidf[kw_vector]]
            for i in range(len(sim)):
                print('keyword is similar to text%d: %.2f' % (i + 1, sim[i]))

        return 0

    @staticmethod
    def get_indicators_user(user_file):
        df_csv = pd.read_csv(user_file)

        tweet_list = df_csv[["text"]]
        tweet_list['text'] = tweet_list['text'].apply(str)

        tweet_list['sentiment'] = tweet_list['text'].apply(get_sentiment_value)
        tweet_list['polarity'] = tweet_list['text'].apply(get_polarity_value)
        tweet_list['subjectivity'] = tweet_list['text'].apply(get_subjectivity_value)

        average_sentiment = tweet_list['sentiment'].mean()
        average_polarity = tweet_list['polarity'].mean()
        average_subjectivity = tweet_list['subjectivity'].mean()
        average_post_time_gap = Indicator.get_mean_time_gap(df_csv['created_at'].values) / 1000 / 60
        if math.isnan(average_post_time_gap) is True:
            average_post_time_gap = 0

        nb_topics = Indicator.nb_different_topics(tweet_list['text'])
        print(user_file)

        return average_sentiment, average_polarity, average_subjectivity, average_post_time_gap, nb_topics, tweet_list['text'].size

    def get_all_indicator_users(self):
        average_sentiment = []
        average_polarity = []
        average_subjectivity = []
        average_time_gap = []
        average_nb_topics = []

        out = pd.DataFrame(columns=['sentiment', 'polarity', 'subjectivity', 'timegap', 'nb_topics', 'nb_tweets'])

        cpt = 1
        for element in os.listdir(self.data_folder):
            if not os.path.isdir(element):
                sentiment, polarity, subjectivity, time_gap, nb_topics, nb_tweets = self.get_indicators_user(os.path.join(self.data_folder, element))
                row = pd.DataFrame(np.array([[sentiment, polarity, subjectivity, time_gap, nb_topics, nb_tweets]]),
                                   columns=['sentiment', 'polarity', 'subjectivity', 'timegap', 'nb_topics', 'nb_tweets'])
                out = pd.concat([out, row], ignore_index=True)

                average_sentiment.append(sentiment)
                average_polarity.append(polarity)
                average_subjectivity.append(subjectivity)
                average_time_gap.append(time_gap)
                average_nb_topics.append(nb_topics)

                cpt += 1

        out.to_csv(self.data_folder+".csv", index=False)

        average_sentiment = self.average(average_sentiment)
        average_polarity = self.average(average_polarity)
        average_subjectivity = self.average(average_subjectivity)

        average_time_gap = np.array(average_time_gap)
        average_time_gap_no_zero = average_time_gap[average_time_gap != 0.0]
        average_time_gap = self.average(average_time_gap_no_zero)

        average_nb_topics = self.average(average_nb_topics)

        return average_sentiment, average_polarity, average_subjectivity, average_time_gap, average_nb_topics

if __name__ == '__main__':
    indi = Indicator('./example/troll-user-tweets')
    print(indi.get_all_indicator_users())
    indi = Indicator('./example/nontroll-user-tweets')
    print(indi.get_all_indicator_users())


import cgitb
import json
import os
import io
import urllib
from urllib.parse import unquote
from .request import Request, RequestMaker
from .response import Response
from .routers import routes, itinerary
from ..core.errors import MuscularError
import cgi
from urllib.parse import unquote
from ..storage import StorageMapper
storageMapper = StorageMapper()

MAX_LINE = 64 * 1024
MAX_HEADERS = 100
TIMEOUT = 2
MAX_CONNECTIONS = 1000

class Transport:
    """
    Транспорт протокола стратегии
    """

    locale = storageMapper.get('locale')
    logger = storageMapper.get('logger')

    server = None

    def __init__(self):
        pass

    def init_server(self, server):
        self.server = server

    def make_response(self, response):
        pass

    def make_request(self):
        pass

class WsgiTransport(Transport):
    """
    Транспорт стратегии WSGI
    """

    locale = storageMapper.get('locale')
    logger = storageMapper.get('logger')

    def execute(self, *args, **kwargs):
        """
        Исполняем условия транспорта
        :param args:
        :param kwargs[environ]: Окружение запроса
        :param kwargs[start_response]: Метод для ответа
        :return:
        """
        self.environ = kwargs['environ']
        self.start_response = kwargs['start_response']
        return self.handler(self.environ, self.start_response)

    def handler(self, environ, start_response):
        """
        Обработчик транспорта

        :param environ: Переменные окружения запроса
        :param start_response: Метод ответа
        :return:
        """
        request = self.make_request(environ)
        if request is None:
            self.logger.debug(self.locale('HTTP Ответ 400 от сервера "Bad request"'))
            raise MuscularError(400, 'Bad request', 'Malformed request line')
        return self.server.handler(request)

    def make_response(self, response):
        """
        Отправляем ответ
        :param response: объект ответа
        :return:
        """
        self.send_header(str(response.http_status), response.headers)
        yield response.body

    def send_header(self, http_status, headers):
        """
        Передаем заголовки ответа
        :param http_status: HTTP статус ответа
        :param headers: Заголовки
        :return:
        """
        return self.start_response(str(http_status), headers)

    def make_request(self, environ):
        """
        Формируем обхект запроса на основании переменных запроса
        :param environ: Переменные запроса
        :return: Request
        """
        requestMaker = RequestMaker(environ)
        request = requestMaker.make()

        self.logger.debug(self.locale('Формируем тело Request'), extra={
            'request': request.__dict__
        })
        return request

#
# class TCPSocketTransport(HttpTransport):
#     __charset = 'iso-8859-1'
#
#     def execute(self, *args, **kwargs):
#         '''
#         :param args[0]: host
#         :param args[1]: port
#         :param kwargs:
#         :return:
#         '''
#         self.host = args[0]
#         self.port = args[1]
#         server = socket.socket(socket.AF_INET, socket.SOCK_STREAM, proto=0)
#         # server.setblocking(0)
#         server.settimeout(TIMEOUT)
#         all_threads = []
#         try:
#             server.bind((self.host, self.port))
#             server.listen(MAX_CONNECTIONS)
#             while True:
#                 try:
#                     connect, _ = server.accept()
#                     t = threading.Thread(target=self.handler, args=(connect,))
#                     t.start()
#
#                     all_threads.append(t)
#                 except ConnectionResetError:
#                     connect = None
#                 except Exception as e:
#                     # self.send_error(connect, e)
#                     traceback.print_exc(file=sys.stdout)
#         finally:
#             if server:
#                 server.close()
#             for t in all_threads:
#                 t.join()
#         return
#
#     def handler(self, connect):
#         try:
#             self.reader = connect.makefile('rb')
#             self.writer = connect.makefile('wb')
#             request = self.make_request()
#             if request is None:
#                 raise MuscularError(400, 'Bad request', 'Malformed request line')
#             self.server.handler(request)
#         except ConnectionResetError:
#             connect = None
#         except Exception as e:
#             # self.send_error(connect, e)
#             traceback.print_exc(file=sys.stdout)
#         if connect:
#             connect.close()
#
#     def make_response(self, response):
#         try:
#             status_line = f'HTTP/1.1 {response.status} {response.reason}\r\n'
#             self.writer.write(status_line.encode('iso-8859-1'))
#
#             if response.headers:
#                 for (key, value) in response.headers:
#                     header_line = f'{key}: {value}\r\n'
#                     self.writer.write(header_line.encode('iso-8859-1'))
#
#             self.writer.write(b'\r\n')
#
#             if response.body:
#                 self.writer.write(response.body)
#
#             if hasattr(self.writer, 'flush'):
#                 self.writer.flush()
#             if self.writer:
#                 self.writer.close()
#         except Exception as err:
#             print('Internal Error:', err)
#             traceback.print_exc(file=sys.stdout)
#         return
#
#     def make_request(self):
#         headers = []
#         size = 0
#         while True:
#             line = self.reader.readline()
#             if len(line) > MAX_LINE:
#                 raise MuscularError(494, 'Request header too large')
#             if b'Content-Length' in line:
#                 s = re.sub("[\r\n]", "", line)
#                 h = s.split(':')
#                 size = h[1]
#             if line in (b'\r\n', b'\n', b''):
#                 break
#             headers.append(line)
#
#         if len(headers[1:]) > MAX_HEADERS:
#             raise MuscularError(494, 'Too many headers')
#         headers = Parser().parsestr(b''.join(headers[1:]).decode(self.__charset))
#
#         words = headers[0].decode(self.__charset).split()
#         if len(words) != 3:
#             raise HTTPErrorMuscularError(400, 'Bad request', 'Malformed request line')
#         method, target, protocol = words
#         if protocol != 'HTTP/1.1':
#             raise HTTPErrorMuscularError(505, 'HTTP Version Not Supported')
#
#         host = self.headers.get('Host')
#         if not host:
#             raise MuscularError(400, 'Bad request', 'Host header is missing')
#
#         if not size:
#             body = b''
#         else:
#             body = self.reader.read(size)
#
#         request = Request(
#             method=method,
#             protocol=protocol,
#             url=target,
#             server=(self.host, self.port),
#             remote_addr=None,
#             headers=headers,
#             body=body,
#         )
#         if self.reader and hasattr(self.reader, 'close'):
#             self.reader.close()
#         return request

class WsgiServer:
    """
    Объект сервера WSGI
    """

    __transport_class = WsgiTransport
    __transport = WsgiTransport
    __host = 'localhost'
    __port = 80

    locale = storageMapper.get('locale')
    logger = storageMapper.get('logger')

    def __init__(self, host, port):
        self.__host = host
        self.__port = port

        self.__transport = self.__transport_class()
        self.__transport.init_server(self)

    def init_transport(self, transport):
        """
        Инициализируем транспорт
        :param transport: Транспорт
        :return:
        """
        self.__transport_class = transport
        self.__transport = transport()
        self.__transport.init_server(self)

    def execute(self, *args, **kwargs):
        """
        Метод исполнения протокола сервера
        :param args:
        :param kwargs:
        :return:
        """
        try:
            return self.__transport.execute(*args, **kwargs)
        except Exception as e:
            return self.send_error(e)

    def handler(self, request):
        """
        Обработчик сервера
        :param request: Запрос к серверу
        :return:
        """
        self.logger.debug(self.locale('Получаем Request(%s)', request), extra={
            'request': request.__dict__
        })

        headers = []
        for header in request.headers:
            headers.append('%s: %s' % (header, request.headers[header]))
        self.logger.info(self.locale("{protocol} {method} {url}",
                                     method=request.method,
                                     url=request.url,
                                     protocol=request.protocol,
                                     ), extra={
            'request': request.__dict__,
            'method': request.method,
            'url': request.url,
            'protocol': request.protocol,
        })
        self.logger.debug(self.locale("{protocol} {method} {url}\t\n {headers}\t\n {body}",
                                      method=request.method,
                                      url=request.url,
                                      protocol=request.protocol,
                                      headers="\n".join(headers),
                                      body=request.body
                                      ), extra={
            'request': request.__dict__,
            'method': request.method,
            'url': request.url,
            'protocol': request.protocol,
            'headers': "\n".join(headers),
            'body': request.body
        })

        static = routes.get_current_static(request)
        if static:
            return self.handle_static(static, request)
        else:
            return self.handle_request(request)

    def handle_request(self, request):
        """
        Обработчик запроса к серверу
        :param request: Объект запроса
        :return:
        """
        for key, instance in itinerary.instance_list():
            call, dictionary = instance.get_current_route(request)
            if call:
                request.route = call
                request.itinerary = instance
                break

        if request.route:
            if request.route['redirect'] and request.route['redirect'] is not None:
                resp = Response.redirect(request.route['redirect'])
            else:
                try:
                    if hasattr(request.route['handler'], 'controller'):
                        resp = request.route['handler'](request.route['handler'].controller(),
                                                        request=request,
                                                        **dictionary)
                    else:
                        resp = request.route['handler'](request=request, **dictionary)
                except MuscularError as me:
                    raise MuscularError(me.status, me.reason, me.body)

                if not isinstance(resp, Response) and isinstance(resp, str):
                    resp = Response.body(200, body=resp)
                elif not isinstance(resp, Response) and isinstance(resp, bytes):
                    resp = Response.body(200, body=resp)
                elif not isinstance(resp, Response) and isinstance(resp, dict):
                    resp = Response.body(200, body=resp)
                elif not isinstance(resp, Response) and isinstance(resp, tuple):
                    kwargs = {}
                    status = 200
                    if len(resp) >= 0:
                        kwargs['body'] = resp[0]
                    if len(resp) >= 1:
                        status = resp[1]
                    if len(resp) >= 2:
                        kwargs['headers'] = resp[2]
                    resp = Response.body(status, **kwargs)
                elif not isinstance(resp, Response):
                    resp = Response.body(200, body=resp)

                if hasattr(request.itinerary, 'modify_response'):
                    resp = request.itinerary.modify_response(resp)

                headers = []
                for header in resp.headers:
                    headers.append('%s: %s' % (header[0], header[1]))
                self.logger.debug(self.locale("Ответ {http_status}\n {headers}\n {body}",
                                              resp=resp,
                                              headers="\n".join(headers),
                                              http_status=resp.http_status,
                                              body=resp.body,
                                              ))
            return self.__transport.make_response(resp)
        self.logger.debug(self.locale("Не удалось найти подходящий роутер в системе"))
        raise MuscularError(404, 'Not found')

    def handle_static(self, static, request):
        """
        Обработчик статических файлов
        :param static: Путь к диреткории с файлами
        :param request: Объект запроса
        :return:
        """
        path = request.path.replace(static['prefix'] + '/', '', 1)
        resp_file = os.path.join(static['directory'], unquote(path))

        if not os.path.isfile(resp_file):
            self.logger.debug(self.locale("Не удалось найти подходящий файл"))
        try:
            resp = Response.file(200, file=resp_file)

            if static['handler'] is not None:
                resp = static['handler'](resp)

            headers = []
            for header in resp.headers:
                headers.append('%s: %s' % (header[0], header[1]))
            self.logger.debug(self.locale("Ответ STATIC {http_status}\n {headers}\n {body}",
                                          resp=resp,
                                          headers="\n".join(headers),
                                          http_status=resp.http_status,
                                          body=resp.body,
                                          ))

            self.__transport.send_header(resp.status, resp.headers)
            with io.open(resp_file, "rb") as f:
                yield f.read()
        except Exception as e:
            self.logger.exception(self.locale("Обработка статического файла закончилась ошибкой"))
            raise MuscularError(404, 'Not found')

    def send_error(self, err):
        """
        Отправляет ответ ошибки
        :param err: Объект ошибки или текст ошибки
        :return:
        """
        try:
            status = err.status if hasattr(err, 'status') else 500
            reason = err.reason if hasattr(err, 'reason') else str(err)
            body = err.body if hasattr(err, 'body') else str(err)
        except Exception as e:
            status = 500
            reason = b'Internal Server Error'
            body = b'Internal Server Error'

        print('Internal Error:', err)
        self.logger.exception(str(err))
        # traceback.print_exc(file=sys.stdout)

        resp = Response.error(status, reason=reason, body=body)
        # call = routes.get_current_error_handler(resp)

        for key, instance in itinerary.instance_list():
            call = instance.get_current_error_handler(resp)
            if call:
                break

        if call is not None:
            resp = call['handler'](resp)
            if not isinstance(resp, Response) and isinstance(resp, str):
                resp = Response.error(status, reason=reason, body=resp)
            elif not isinstance(resp, Response) and isinstance(resp, bytes):
                resp = Response.error(status, reason=reason, body=resp)
            elif not isinstance(resp, Response) and isinstance(resp, dict):
                resp = Response.error(status, reason=reason, body=resp)
            elif not isinstance(resp, Response) and isinstance(resp, tuple):
                kwargs = {}
                if len(resp) >= 0:
                    kwargs['reason'] = resp[0]
                if len(resp) >= 1:
                    status = resp[1]
                if len(resp) >= 2:
                    kwargs['headers'] = resp[2]
                resp = Response.error(status, **kwargs)
            elif not isinstance(resp, Response):
                resp = Response.error(status, reason=reason, body=resp)

            headers = []
            for header in resp.headers:
                headers.append('%s: %s' % (header[0], header[1]))
            self.logger.debug(self.locale("Ответ ERROR {http_status}\n {headers}\n {body}",
                                          resp=resp,
                                          headers="\n".join(headers),
                                          http_status=resp.http_status,
                                          body=resp.body,
                                          ))
        return self.__transport.make_response(resp)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Jul 13 14:19:34 2022

@author: ali
"""

import dcor
import math

import numpy as np
from timeit import timeit
import matplotlib.pyplot as plt
from numpy import random
import time

def vertex(x, y, delta=0.05, z=None, return_plot = False, verbose=False):

    V=np.arange(0, x.shape[2], 1, dtype=int)

    V_temp=V
    V_dim=V_temp.shape
    V_dim,=V_dim
    x_reduced=x

    dcors=[]
    V_temp_all = []
    k=0
    V_temp_all = []
    c_all=[]
    V_temp_all.append(V)
    if verbose:
        print('Begin iterating process')
    while V_dim > 1:
        time1= time.time()
        c=[]
        V_index=[]
        for u in V_temp:
            xtemp=x[:,V_temp,:]
            #print(xtemp.shape)
            xtemp=xtemp[:,:,u]
            #print(xtemp)
            #print(dcor.distance_correlation(xtemp, y))
            if z is None:
                c=np.append(c,dcor.distance_correlation(xtemp, y))
            else:
                c=np.append(c,dcor.partial_distance_correlation(xtemp, y,z)) #partial distance correlation

        dcors.append(np.max(c))
        t_temp=np.quantile(c,delta)
        index_temp=c>t_temp

        V_temp = np.array(V_temp_all[k])[index_temp]
        V_temp_all.append(V_temp)
        c_all.append(c)
        V_dim=np.size(V_temp)
        time_step = time.time() - time1
        if verbose:
            print(f'Step {k} accomplished in {np.round(time_step)} secs, max time {np.round(x.shape[1]*time_step*(1-delta)**k)}')
        k=k+1

    index_winner=np.max(np.argwhere(dcors == np.amax(dcors)))

    if verbose:
        print(dcors[index_winner])
        print(V_temp_all[index_winner].shape)

    V_winner=V_temp_all[index_winner]
    c_winner=c_all[index_winner]

    if return_plot:
        import matplotlib.pyplot as plt
        dimensions=[x.shape[0]for x in V_temp_all]
        plt.scatter( dimensions[0:len(dimensions)-1] , dcors )
        plt.title("Signal Distance Correlation")
        plt.xlabel("Dimension of Vertices")
        plt.ylabel("Maximum Distance Correlation")
        plt.savefig('/Users/daniel/Downloads/algo1fig.png', dpi=1000)
        return V_winner, c_winner, dcors, plt
    else:
        return V_winner, c_winner, dcors


import logging
import os
import time

import requests
from dotenv import load_dotenv
from telegram import Bot

logger = logging.getLogger("Devman Bot")

class TelegramHandler(logging.Handler):
    def __init__(self, tg_bot_token, chat_id, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.bot = Bot(token=tg_bot_token)
        self.chat_id = chat_id

    def emit(self, record):
        log_entry = self.format(record)
        self.bot.send_message(chat_id=self.chat_id, text=log_entry)

def get_new_attempts(token, timestamp=None):
    dvmn_url = "https://dvmn.org/api/long_polling/"
    headers = {
        "Authorization": f"Token {token}",
    }
    params = {
        "timestamp": timestamp,
    }
    response = requests.get(
        dvmn_url,
        headers=headers,
        params=params,
        timeout=60
    )
    response.raise_for_status()
    return response.json()

def main():
    load_dotenv()

    tg_token = os.getenv('TG_TOKEN')
    tg_chat_id = os.getenv('TG_CHAT_ID')
    dvmn_api_token = os.getenv('DVMN_API_TOKEN')

    bot = Bot(token=tg_token)

    logging_bot_token = os.getenv('LOGGING_BOT_TOKEN')
    logger.setLevel(logging.INFO)

    formatter = logging.Formatter(
        '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
    )

    tg_handler = TelegramHandler(logging_bot_token, tg_chat_id)
    tg_handler.setLevel(logging.INFO)
    tg_handler.setFormatter(formatter)

    logger.addHandler(tg_handler)

    last_timestamp = None
    logger.info("Бот запущен")
    while True:
        try:
            logger.debug("Making request to Devman API...")
            attempts = get_new_attempts(dvmn_api_token, last_timestamp)
            logger.debug("Received response from Devman API.")

            if attempts['status'] == 'found':
                new_attempts = attempts['new_attempts']
                for attempt in new_attempts:
                    is_negative = attempt['is_negative']
                    lesson_title = attempt['lesson_title']
                    lesson_url = attempt['lesson_url']
                    bot.send_message(
                        chat_id=tg_chat_id,
                        text=f"Урок '{lesson_title}' проверен. Результат: {'неудачно' if is_negative else 'успешно'}. Ссылка на урок {lesson_url}",
                    )
                    logger.info("Sent message to Telegram.")
                    last_timestamp = attempt['timestamp']
            else:
                last_timestamp = attempts['timestamp_to_request']

        except requests.exceptions.ReadTimeout:
            logger.debug("Read timeout occurred. The server did not respond in a timely manner. Retrying immediately")
        except requests.exceptions.ConnectionError:
            logger.warning("A connection error occurred. Please check the network connection. Retrying in 10 minutes...")
            time.sleep(600)
        except Exception:
            logger.exception("An unexpected error occurred.")

if __name__ == "__main__":
    main()


import chardet
import random
import hashlib
import binascii
import base58

def load_server_list(filename):
    with open(filename) as f:
        servers = ['http://{0}'.format(url.strip()) for url in f.readlines()]
        random.shuffle(servers)
        return servers

def detect_webpage_encoding(html):
    result = chardet.detect(html)
    return result['encoding']

def validate_bitcoin_wallet(bitcoin):
    base58decoder = base58.b58decode(bitcoin).hex()
    prefixandhash = base58decoder[:len(base58decoder)-8]
    checksum = base58decoder[len(base58decoder)-8:]
    hash = prefixandhash
    for x in range(1,3):
        hash = hashlib.sha256(binascii.unhexlify(hash)).hexdigest()
    if checksum == hash[:8]:
        return True
    else:
        return False


#!/usr/bin/python3
from flask import Flask, render_template, request, url_for, redirect
from werkzeug.utils import secure_filename
from werkzeug.datastructures import  FileStorage
import os
from generator import Generator
from encoder import Encoder
from led import Led
from font import Font
app = Flask(__name__, template_folder='./html')

uploads_dir = os.path.join('.', 'frames')
os.makedirs(uploads_dir, exist_ok=True)

num_lights = 100
num_colors = 250
screen_height = 5
DEFAULT_FPS = 60
DEFAULT_NUM_FRAMES = 250

board = Led(num_lights, num_colors)
board.run()

def make_tree(path):
    tree = dict(name=path, children=[])
    try: lst = os.listdir(path)
    except OSError:
        pass #ignore errors
    else:
        for name in lst:
            fn = os.path.join(path, name)
            if os.path.isdir(fn):
                tree['children'].append(make_tree(fn))
            else:
                tree['children'].append(dict(name=fn))
    return tree

@app.route('/upload')
def upload_file():
   return render_template('upload.html')

@app.route('/uploader', methods = ['GET', 'POST'])
def upload_file_route():
	if request.method == 'POST':
		f = request.files['file']
		f.save(os.path.join(uploads_dir, secure_filename(f.filename)))
		return 'file uploaded successfully'
	elif request.method == 'GET':
		return render_template('upload.html')

def get_uploaded_files():
	children = make_tree(uploads_dir)["children"]
	files = []
	for child in children:
		files.append(child["name"].split('/')[-1])
	return files

@app.route('/play', methods = ['GET', 'POST'])
def play():
	files = get_uploaded_files()
	if request.method == 'POST':
		f = request.form["file"]
		fps = int(request.form["fps"])
		with open(os.path.join(uploads_dir, f), "rb") as frames:
			board.put(frames.read(), fps)
		return render_template('play.html', data=files)
	elif request.method == 'GET':
		if "func" in request.args:
			play_generated_frames(parse_args(request.args), request.args['func'])
		return render_template('play.html', data=files)

@app.route('/text', methods = ['GET', 'POST'])
def text():
	if request.method == 'POST':
		font = Font(screen_height, int(num_lights / screen_height))
		e = Encoder(num_lights, num_colors)
		board.put(e.encode(font.text_to_frames(request.form['text'])))
		return render_template('text.html')
	else:
		return render_template('text.html')

def play_generated_frames(args, func):
	g = Generator(num_lights, num_colors, screen_height)
	e = Encoder(num_lights, num_colors)

	frames = None
	if func == "rainbow":
		frames = g.generate_rainbow_frames(args['num_frames'])
	elif func == "flow":
		frames = g.generate_flow_frames(args['num_frames'])
	elif func == "dot":
		frames = g.generate_dot_frames(args['num_frames'])
	elif func == "radiate":
		frames = g.generate_radiate_frames(args['num_frames'])
	elif func == "cascade":
		frames = g.generate_cascade_frames()
	elif func == "snake":
		frames = g.generate_snake_frames()
	elif func == "game_of_life":
		frames = g.generate_game_of_life_frames(args['num_frames'])
	elif func == "dance":
		frames = g.generate_dance_frames(args['num_frames'])
	elif func == "merge" or func == "quick" or func == "comb":
		frames = g.generate_sort_frames(func)

	if frames is not None:
		board.put(e.encode(frames), args['fps'])

def parse_args(args):
	fps = DEFAULT_FPS
	num_frames = DEFAULT_NUM_FRAMES
	if 'fps' in args:
		try:
			fps = int(args['fps'])
		except ValueError:
			pass
	if 'num_frames' in args:
		try:
			num_frames = int(args['num_frames'])
		except ValueError:
			pass

	return {'fps': fps, 'num_frames': num_frames}

if __name__ == '__main__':
   app.run(host='0.0.0.0',debug = True)