import aiohttp
import asyncio
import json

from . import errors

async def json_or_text(response):
    text = await response.text(encoding="utf-8")
    try:
        if "application/json" in response.headers["Content-Type"]:
            return json.loads(text)
    except KeyError:
        # Thanks Cloudflare
        pass

    return text

class Route:
    BASE = "https://g.tenor.com/v1"

    def __init__(self, path):
        self.url = f"{self.BASE}{path}"

class HTTPClient:
    def __init__(self, api_key):
        self.api_key = api_key
        self.session = None

    async def request(self, route, parameters=None):
        if parameters is None:
            parameters = {}
        parameters["key"] = self.api_key

        if self.session is None:
            self.session = await self._create_session()

        params = {}
        for k, v in parameters.items():
            if v is not None:
                params[k] = v

        for tries in range(5):
            async with self.session.get(route.url, params=params) as resp:
                data = await json_or_text(resp)

                if "error" in data and resp.status == 200:
                    raise errors.TenorException(data["error"])

                if 200 <= resp.status < 300:
                    return data

                if resp.status == 404:
                    raise errors.NotFound(data)

                if resp.status == 429:
                    # we are rate limited, wait for 30s
                    await asyncio.sleep(30)
                    continue

                return data

        if resp.status >= 500:
            raise errors.TenorServerError(data)

    async def _create_session(self):
        return aiohttp.ClientSession()

    async def close(self):
        if self.session:
            await self.session.close()

    async def search(self, **parameters):
        return await self.request(Route("/search"),
                                  parameters=parameters)

    async def trending(self, **parameters):
        return await self.request(Route("/trending"),
                                  parameters=parameters)

    async def categories(self, **parameters):
        return await self.request(Route("/categories"),
                                  parameters=parameters)

    async def search_suggestions(self, **parameters):
        return await self.request(Route("/search_suggestions"),
                                  parameters=parameters)

    async def autocomplete(self, **parameters):
        return await self.request(Route("/autocomplete"),
                                  parameters=parameters)

    async def trending_terms(self, **parameters):
        return await self.request(Route("/trending_terms"),
                                  parameters=parameters)

    async def registershare(self, **parameters):
        return await self.request(Route("/registershare"),
                                  parameters=parameters)

    async def gifs(self, **parameters):
        return await self.request(Route("/gifs"),
                                  parameters=parameters)

    async def random(self, **parameters):
        return await self.request(Route("/random"),
                                  parameters=parameters)

    async def anonid(self, **parameters):
        return await self.request(Route("/anonid"),
                                  parameters=parameters)


from typing import Dict, List, Union, Tuple
from urllib.parse import urljoin

import requests
from PIL import Image
from bs4 import BeautifulSoup
from email_validator import validate_email, EmailNotValidError

from plan_ilan.data_mining.staff.lookup_parameters import StaffLookup, StaffLookupAnswer
from plan_ilan.apps.web_site.models import DepartmentEnum

class StaffCrawler:
    """An object that crawls over different staff members web pages according to their attributes."""

    def __init__(self, url: Union[str, List[str]], department: DepartmentEnum, staff_lookup: StaffLookup,
                 **kwargs) -> None:
        self._urls = url if isinstance(url, List) else [url]
        self._department = department
        self.staff_lookup = staff_lookup
        self._has_photos = kwargs['has_photos'] if 'has_photos' in kwargs else True
        self._email_suffix = kwargs['email_suffix'] if 'email_suffix' in kwargs else ''
        self.__teachers_data = None

    @property
    def urls(self) -> List[str]:
        """Return the url of the crawled web page"""
        return self._urls

    @property
    def department(self):
        """Returns the department of the crawled web page"""
        return self._department

    def get_teachers_data(self) -> List[StaffLookupAnswer]:
        if self.__teachers_data is None:
            self.crawl()
        return self.__teachers_data

    def crawl(self) -> int:
        """"The main method for this interface, crawls over the staff members web page.
         returns the amount of valid entries found."""
        self.__teachers_data = []
        for url in self.urls:
            response = requests.get(url)
            if not response:
                continue
            soup = BeautifulSoup(response.text, 'html.parser')
            persons = self.staff_lookup.persons.get_tags(soup)
            for person in persons:
                teacher_data = StaffLookupAnswer()
                details_url = self.staff_lookup.details_url.get_single_tag(person)
                if details_url is None:
                    continue
                details_url = details_url['href'].strip()
                teacher_data.title = self.staff_lookup.params.title.get_values_from_tag(person)
                teacher_data.name = self.staff_lookup.params.name.get_values_from_tag(person)
                if not teacher_data.is_valid:
                    continue
                teacher_response = requests.get(details_url)
                if not teacher_response:
                    continue
                teacher_web_page = BeautifulSoup(teacher_response.text, 'html.parser')
                teacher_data.email = self.staff_lookup.params.email.get_values_from_tag(teacher_web_page)
                if not teacher_data.email:
                    teacher_data.email = ''
                mail_valid, value = self._validate_email(teacher_data.email)
                if mail_valid:
                    teacher_data.email = value
                else:
                    # todo: add logging
                    print(teacher_data.email, value)
                teacher_data.phone = self.staff_lookup.params.phone.get_values_from_tag(teacher_web_page)
                teacher_data.website = self.staff_lookup.params.website.get_values_from_tag(teacher_web_page)
                teacher_data.office = self.staff_lookup.params.office.get_values_from_tag(teacher_web_page)
                image_tag = self.staff_lookup.params.photo.get_single_tag(teacher_web_page)
                image_url = urljoin(url, image_tag['src'])
                teacher_data.photo = Image.open(requests.get(image_url, stream=True).raw)
                self.__teachers_data.append(teacher_data)
        return len(self.__teachers_data)

    def _validate_email(self, email: str) -> Tuple[bool, Union[Dict, str]]:
        """Validate the given email address
        Args:
            email:str
                The email to validate
        Returns:
            tuple -
                A tuple of size 2:
                1) outcome - indicates whether the email address is valid
                2) value - if outcome is true returns the email address in normalized form, else
                           returns a human readable error message.
        """
        try:
            if '@' not in email:
                email = f'{email}{self._email_suffix}'
            valid = validate_email(email, timeout=10)
            outcome = True
            value = valid.email
        except EmailNotValidError as e:
            outcome = False
            value = str(e)
        return outcome, value


import qiskit as qk
from qiskit import QuantumProgram

qp = QuantumProgram()
qr = qp.create_quantum_register('qr', 2)
cr = qp.create_classical_register('cr',2)
qc = qp.create_circuit('qc', [qr], [cr])
circuit = qp.get_circuit('qc')
quantum_r = qp.get_quantum_register('qr')
classical_r = qp.get_classical_register('cr')

circuit.x(quantum_r[0])
circuit.y(quantum_r[1])

circuit.cx(quantum_r[0], quantum_r[1])

circuit.measure(quantum_r[0], classical_r[0])
circuit.measure(quantum_r[1], classical_r[1])

backend = 'local_qsam_simulator'
circuits = ['qc']

result = qp.execute(circuit, backend, wait=2, timeout = 240)
print(result)
result.get_counts('qc')
out = result.get_ran_qasm('qc')
print(out)


import time
from selenium import webdriver
from selenium.webdriver.common.by import By
from webdriver_manager.chrome import ChromeDriverManager
from bs4 import BeautifulSoup
import csv
import random

linkas = input("Autopliuso masinu linkas: ")

def randomlaikas():
    _sleep = random.randint(3,10)
    time.sleep(_sleep)

def scrape_phone_numbers():
    driver = webdriver.Chrome(executable_path=ChromeDriverManager().install())
    driver.get(linkas)
    time.sleep(4)
    accept_cookies = driver.find_element(By.XPATH, '//button[text()="Sutinku"]')
    accept_cookies.click()
    time.sleep(3)

    results = []
    num_cars = int(input("Kiek nuskreipinti automobiliu: "))
    current_window_handle = driver.current_window_handle

    while len(results) < num_cars:
        cars = driver.find_elements(By.XPATH, '//div[@class="announcement-title"]')

        for i, car in enumerate(cars):
            if i == num_cars:
                break
            current_window_handle = driver.current_window_handle
            car.click()
            time.sleep(3)
            for handle in driver.window_handles:
                if handle != current_window_handle:
                    driver.switch_to.window(handle)
            html = driver.page_source
            soup = BeautifulSoup(html, "html.parser")
            phone = soup.find("div", class_="button seller-phone-number js-phone-number")
            if phone is not None:
                phone_number = phone.text.replace(" ", "")
                if "+370" in phone_number and phone_number not in results:
                    results.append(phone_number)
            driver.close()
            driver.switch_to.window(current_window_handle)

        try:
            next_page = driver.find_element(By.XPATH, '//a[@rel="next" and @class="next"]')
            next_page.click()
            time.sleep(3)
        except Exception as e:
            break

    with open("phones.csv", "w", newline="") as f:
        writer = csv.writer(f)
        writer.writerow(["Phone Number"])
        for phone_number in results:
            phone_number = phone_number.strip('"\n')
            if "+370" not in phone_number:
                continue
            writer.writerow([phone_number])
        print(results)

scrape_phone_numbers()


from io import BytesIO
import re
import sys
import requests
import img2pdf
import PyPDF4
from rich.console import Console
from rich.progress import track

console = Console()

INPUT_SESSION = console.input("🍪  Your '_reader_session' key: ")
ENDPOINT_SCUOLABOOK = "https://webapp.scuolabook.it/books"
HEADER = {'X-Requested-With': 'XMLHttpRequest',
          'Accept': 'application/json, text/javascript, */*; q=0.01',
          'Cookie': f'_reader_session={INPUT_SESSION}'}

class Book:
    """
    A Book interface
    """

    def __init__(self):
        self.book_id: int = 0
        self.title: bytes = ''
        self.author: bytes = ''
        self.publisher: bytes = ''
        self.isbn: bytes = ''
        self.npages: int = 0

def get_books_list_data():
    """
    This function return a dictionary that contains all books present in your library
    """
    data = requests.get(ENDPOINT_SCUOLABOOK,
                        headers=HEADER).text
    books = {}
    for i in range(len(re.findall('"id":(.*?),', data))):
        books[re.findall('"id":(.*?),', data)[i]] = re.findall(
            '"ws_title":"(.*?)"', data)[i].replace("\u0026", "&")
    return books

def get_books_list(books_list_data: dict):
    """
    This function takes in input the `books_list_data` dictionary data that contains all books
    check if the `_reader_session` is valid and if don't exit the program.
    Format them in a readable user manner and output them
    so the user can choose what book wanna download
    """
    if books_list_data == {}:
        console.print(
            "❌  Your library is empty or your '_reader_session' key is expired")
        exit(255)
    console.print("\n📚  [b][Your library][/b]\n")
    for bookid in books_list_data:
        console.print(f"📗  ID:({bookid})  [b]{books_list_data[bookid]}[/b]")

def get_selected_book(books_list_data: dict):
    """
    This function takes in input the `books_list_data` dictionary data that contains all books
    check if the `input_book_id` is valid and  present in the library if don't exit the program.
    Output to the user all the information about the book selected and return a book object
    """
    try:
        input_book_id = int(console.input(
            "\n🔢  Insert ID of the book to download them: "))
        for bookid in books_list_data:
            if not input_book_id != bookid:
                raise ValueError
    except ValueError:
        console.print("❌  Invalid Book ID")
        exit(255)
    book = Book()
    data = requests.get(
        f"{ENDPOINT_SCUOLABOOK}/{input_book_id}", headers=HEADER).text
    book.book_id = input_book_id
    book.title = re.search('"ws_title":"(.*?)"', data).group(1).encode('utf-8')
    book.author = re.search('"ws_author":"(.*?)"',
                            data).group(1).encode('utf-8')
    book.publisher = re.search(
        '"ws_publisher":"(.*?)"', data).group(1).encode('utf-8')
    book.isbn = re.search('"ws_isbn":"(.*?)"', data)
    if book.isbn:
        book.isbn = book.isbn.group(1).encode('utf-8')
    book.npages = int(re.search('"ws_num_pages":"(.*?)"',
                      data).group(1).encode('utf-8'))
    console.print("\n[Selected book]\n")
    console.print(f"📖  Title: {book.title.decode()}")
    console.print(f"👤  Author: {book.author.decode()}")
    console.print(f"🏛  Publisher: {book.publisher.decode()}")
    console.print(f"📜  ISBN: {book.isbn.decode()}")
    console.print(f"📄  Pages: {book.npages}")
    return book

def get_all_pages(book: Book):
    """
    This function take in input a `Book` object  and return a list of page link to download
    """
    data  = ""
    payload = ""
    for i in track(range(1, book.npages + 2)):
        if i % 100 == 0 or i == book.npages + 1:
            data += requests.get(
                f"{ENDPOINT_SCUOLABOOK}/{book.book_id}/pages?{payload[1:]}", headers=HEADER).text
            payload = ""
        payload += f"&pages[]={i}"
    matches = re.findall('":"(.*?)"', data)
    return matches

def dowload_convert_all_pages(book: Book, matches: list):
    """
    This function takes as input a `Book` object and a list of links of pages
    and return a `list` that contains each PDF page in bytes format
    """
    pdfs_bytes = []
    for match in matches:
        console.print(
            f"\r📥  Downloading & Converting page {matches.index(match) + 1}/{book.npages}...")
        sys.stdout.flush()
        link = match.replace("\\u0026", "&").encode('utf-8')
        pdf_bytes = img2pdf.convert(requests.get(link).content)
        pdfs_bytes.append(pdf_bytes)
    return pdfs_bytes

def merging_pdf(pdfs_bytes: list, book: Book):
    """
    This function takes as input a `list` of PDF page of book in bytes format and `Book` object
    each element of the `list` is read and merged with the previous one and at the end of the `list`
    the final merged Book (PDF) is created
    """
    merger = PyPDF4.PdfFileMerger(strict=False)
    output_file = book.title.decode().replace('\u007C', '') + ".pdf"
    console.print("\n☯  Merging everything...")
    for pdf_bytes in pdfs_bytes:
        with BytesIO(pdf_bytes) as pdf_file:
            pdf_file_stream = PyPDF4.PdfFileReader(
                stream=pdf_file, strict=False)
            merger.append(pdf_file_stream)
    console.print(f"⌛  Creating merged book ''{output_file}''...")
    merger.write(output_file)
    console.print(
        f":thumbs_up:  The book: ''{output_file}'' was created succesfully!")

def main():
    """
    This function execute all program
    """
    books_list_data = get_books_list_data()
    get_books_list(books_list_data=books_list_data)
    book = get_selected_book(books_list_data=books_list_data)
    matches = get_all_pages(book=book)
    pdfs_bytes = dowload_convert_all_pages(book=book, matches=matches)
    merging_pdf(pdfs_bytes=pdfs_bytes, book=book)

if __name__ == '__main__':
    main()


import json
import logging
import os
import random
import re
import time
from datetime import datetime

import boto3
import requests
from dateutil import tz
from dateutil.parser import parse
from selectolax.parser import HTMLParser
from utils import headers

logger = logging.getLogger()
logger.setLevel(logging.INFO)

dynamodb = boto3.resource('dynamodb')
table = dynamodb.Table(os.environ['VLR_MATCHES_TABLE'])

# vlr match events cache
vlr_events_cache = {}

def insert(table, matches):
    '''
    put items into specified DynamoDB table.
    '''
    with table.batch_writer() as batch:
        for match in matches:
            logger.info('put match info into the table: {}'.format(match))
            batch.put_item({k: v for k, v in match.items()})

def sleep():
    '''
    sleep for 1~10 secs (randomly)
    '''
    sec = random.randint(1, 10)
    time.sleep(sec)

def get_event_from_cache(event_url_path):
    global vlr_events_cache

    result = ''

    if event_url_path in vlr_events_cache:
        result = vlr_events_cache[event_url_path]

    return result

def scrape_event(event_url_path):
    '''
    scrape event page of url_path
    '''
    global vlr_events_cache

    url = 'https://www.vlr.gg{}'.format(event_url_path)
    logger.info('get event info: {}'.format(url))

    resp = requests.get(url, headers=headers)
    html = HTMLParser(resp.text)

    event_id = int(event_url_path.split('/')[2])

    event_name = html.css_first('.wf-title').text().strip()
    event_name = event_name.replace('\t', '').replace('\n', '')

    country_flag = html.css_first(
        '.event-desc-item-value .flag').attributes['class']
    country_flag = country_flag.replace(' mod-', '_').replace('flag_', '')

    data = {
        'event_id': event_id,
        'event_name': event_name,
        'country_flag': country_flag
    }

    # caching
    vlr_events_cache[event_url_path] = data

    return data

def scrape_match(match_url_path):
    '''
    scrape match page of url_path
    '''
    global vlr_events_cache

    url = 'https://www.vlr.gg{}'.format(match_url_path)
    logger.info('get match info: {}'.format(url))

    resp = requests.get(url, headers=headers)
    html = HTMLParser(resp.text)

    match_id = int(match_url_path.split('/')[1])

    match_name = html.css_first('.match-header-event-series').text()
    match_name = match_name.replace('\t', '').replace('\n', '')

    start_time = html.css_first('.moment-tz-convert').attributes['data-utc-ts']
    with_timezone = ' '.join([start_time, 'EST'])

    tzinfo = {'EST': tz.gettz('America/New_York'),
              'CST': tz.gettz('America/Chicago')}
    start_time_est = parse(with_timezone, tzinfos=tzinfo)
    start_time_utc = start_time_est.astimezone(tz.gettz('Etc/GMT'))
    start_time_utc = datetime.strftime(start_time_utc, '%Y-%m-%dT%H:%M:%S%z')

    teams = html.css('.wf-title-med')
    teams = [t.text().replace('\t', '').replace('\n', '') for t in teams]

    best_of = html.css('.match-header-vs-note')[-1].text()
    best_of = best_of.replace('Bo', '').replace(' Maps', '')
    best_of = best_of.replace('\t', '').replace('\n', '')
    best_of = int(best_of)

    event_url_path = html.css_first('a.match-header-event').attributes['href']

    if event_url_path in vlr_events_cache:
        logger.info('get event info from cache: {}'.format(event_url_path))
        event_info = vlr_events_cache[event_url_path]
    else:
        logger.info('get event info from website: {}'.format(event_url_path))
        event_info = scrape_event(event_url_path)

    data = {
        'match_id': match_id,
        'event_name': event_info['event_name'],
        'event_country_flag': event_info['country_flag'],
        'start_time': start_time_utc,
        'best_of': best_of,
        'match_name': match_name,
        'teams': teams
    }
    return data

def scrape_matches(page: str = 1):
    '''
    scrape /matches page
    '''
    url = 'https://www.vlr.gg/matches?page={}'.format(page)
    logger.info('fetch matches list from: {}'.format(url))

    resp = requests.get(url, headers=headers)
    html = HTMLParser(resp.text)

    matches = []

    for item in html.css('a.wf-module-item'):
        match_url_path = item.attributes['href']

        sleep()
        match_detail = scrape_match(match_url_path)

        item = {
            'id': match_detail['match_id'],
            'eventName': match_detail['event_name'],
            'eventCountryFlag': match_detail['event_country_flag'],
            'startTime': match_detail['start_time'],
            'bestOf': match_detail['best_of'],
            'matchName': match_detail['match_name'],
            'teams': [{'title': team} for team in match_detail['teams']],
            'pagePath': match_url_path
        }
        logger.info('add match to the list: {}'.format(item))
        matches.append(item)

    return matches

def lambda_handler(event, context):
    records = event['Records']
    match_list = []

    for record in records:
        body = json.loads(record['body'])
        page = str(body['page'])

        matches = scrape_matches(page)
        match_list.extend(matches)

    insert(table, match_list)

    return {
        'matches_count': len(match_list)
    }


# -*- coding: utf-8 -*-
"""
Created on Sun Apr 25 19:52:38 2021

@author: Eliu
"""

#pip install keyboard

import keyboard
from time import sleep

def free_fire(cx, cy, w):
    res_x = 640
    res_y = 480
    time_release = 0.1
    sleep(0.001)

    no_move = w/3 #Divisor hace efecto en la sensibilidad
    if cx>(res_x/2)+no_move and cx !=0:
        print("IZQUIERDA")
        keyboard.press('a')
        #sleep(time_release)
        keyboard.press_and_release('d')

    elif cx<(res_x/2)-no_move:
        print("DERECHA")
        keyboard.press('d')
        #sleep(time_release)
        keyboard.press_and_release('a')

    else:
        print('CENTER')
        keyboard.press_and_release('d')
        keyboard.press_and_release('a')

    '''
    keyboard.press('w')
    sleep(1)
    keyboard.press_and_release('w')
    '''

import cv2

face_cascade = cv2.CascadeClassifier('haarcascade_frontalface_default.xml')
cap = cv2.VideoCapture(0) #My camer 640x480

while True:
    _, img = cap.read()
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    faces = face_cascade.detectMultiScale(gray, 1.1,4)
    for (x,y,w,h) in faces:
        cv2.rectangle(img, (x,y), (x+w, y+h), (255,0,0), 2)
        cx = x+(w/2)
        cy = y+(h/2)
        free_fire(cx, cy, w)
        print(str(cx)+'    '+str(cy))
    cv2.imshow('img', img)
    k = cv2.waitKey(30)
    if k == 27:
        break
cap.release()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Nov 18 08:53:40 2020

@author: Mathew
"""
# These are the packages we are using.

from skimage.io import imread
import matplotlib.pyplot as plt
from skimage import filters,measure

image=imread("/Users/Mathew/Desktop/Axioscan/apt.tif")

# This is just to show a small region of the image, since it's difficult to see what's happening with the full image.
region=image[500:1000,500:1000]
plt.imshow(region,cmap="Blues")
plt.show()

# Determine a threshold using the Otsu method - Note, if I was comparing cases etc., I'd keep the threhsold constant, and not use Otsu for all of them.
threshold=filters.threshold_otsu(image)
print(threshold)            # This prints the threshold that Otsu has determined.

filtered=image>threshold    # Apply the threshold to the image. If going for constant threshold for all images, could just replace this with a number.

region_filtered=filtered[:,:]   # Can set a region to look at.
plt.imshow(region_filtered,cmap="Blues")
plt.show()

# Now find the different features in the thresholded image.
label_image=measure.label(region_filtered)
plt.imshow(label_image)
plt.show()

# Measure parameters of labelled regions.
table=measure.regionprops_table(label_image,properties=('area','centroid','orientation','major_axis_length','minor_axis_length'))

# Get the area and length data.
areas=table['area']
lengths=table['major_axis_length']

number=len(areas)  # Count the number of features detected.

print(number)

# Plot some histograms.

plt.hist(areas, bins = 50,range=[0,100], rwidth=0.9,color='#607c8e')
plt.xlabel('Area (pixels)')
plt.ylabel('Number of Features')
plt.title('Area of features')
plt.show()

plt.hist(lengths, bins = 50,range=[0,200], rwidth=0.9,color='#607c8e')
plt.xlabel('Length (pixels)')
plt.ylabel('Number of Features')
plt.title('Length')
plt.show()

# For control:

imagecont=imread("/Users/Mathew/Desktop/Axioscan/noapt.tif")

filteredcont=imagecont>threshold    # Apply the threshold to the image. If going for constant threshold for all images, could just replace this with a number.

label_image_cont=measure.label(filteredcont)

# Measure parameters of labelled regions.
tablecont=measure.regionprops_table(label_image_cont,properties=('area','centroid','orientation','major_axis_length','minor_axis_length'))

# Get the area and length data.
areascont=tablecont['area']
lengthscont=tablecont['major_axis_length']

numbercont=len(areascont)  # Count the number of features detected.

print(numbercont)

# Plot some histograms.

plt.hist(areascont, bins = 50,range=[0,100], rwidth=0.9,color='#607c8e')
plt.xlabel('Area (pixels)')
plt.ylabel('Number of Features')
plt.title('Area of features (control)')
plt.show()

plt.hist(lengthscont, bins = 50,range=[0,200], rwidth=0.9,color='#607c8e')
plt.xlabel('Length (pixels)')
plt.ylabel('Number of Features (control)')
plt.title('Length')
plt.show()


from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import Column, Integer, String, BLOB, DateTime

base = declarative_base()

class ConfigNmap(base):
    __tablename__ = 'config_nmap'
    id = Column(Integer, primary_key=True)
    property = Column(String(255), nullable=False)
    value = Column(String(255), nullable=False)

class NmapReportsDHCPDiscover(base):
    __tablename__ = 'nmap_reports_dhcp_discover'
    id = Column(Integer, primary_key=True)
    time = Column(DateTime)
    report = Column(BLOB)

class NmapReportsSnifferDetect(base):
    __tablename__ = 'nmap_reports_sniffer_detect'
    id = Column(Integer, primary_key=True)
    time = Column(DateTime)
    report = Column(BLOB)

class NmapDiff(base):
    __tablename__ = 'nmap_diff'
    id = Column(Integer, primary_key=True)
    result = Column(BLOB)


import os
import tensorflow as tf
import time
import numpy as np
import data_loader
import matplotlib.pyplot as plt
from tqdm import trange

class DNN:
    """DNN模型训练"""

    def __init__(self, path=None, sheet_name="Sheet2",
                 save_model_path="./model/model-1/", batch_size=32,
                 learning_rate=0.001, num_steps=10000, model_dim=(101, 64, 40),
                 steps=10, threshold_value=0.9, dropout=0, save_model_threshold_value=0.8):
        """
        创建模型训练实例
        :param path: 输入xlsx文件路径
        :param save_model_path: 模型保存路径
        :param batch_size: 训练batch大小
        :param num_steps: 迭代步数
        :param steps: 打印评估间隔步数
        :param threshold_value: 输出阈值
        :param dropout: 丢弃率
        :param save_model_threshold_value: 模型保存阈值
        """
        self.path = path
        self.save_model_path = save_model_path
        self.batch_size = batch_size
        self.num_steps = num_steps
        self.steps = steps
        self.all_counts = int(num_steps / steps) - 1
        self.count = 0  # 评估计数
        self.fig_loss = np.zeros([self.all_counts])
        self.fig_acc = np.zeros([self.all_counts])
        self.save_model_threshold_value = save_model_threshold_value  # 保存模型阈值
        self.threshold_value = threshold_value  # 阈值
        self.sheet_name = sheet_name
        self.keep_rate = 1 - dropout  # dropout使神经网络层中随机丢弃神经单元的值的保留率
        self.learning_rate = learning_rate  # 学习率
        self.hidden_dim = model_dim[1:-1]   # 隐藏层维度
        self.input_dim = model_dim[0]   #输入层维度
        self.output_dim = model_dim[-1] #输出层维度

        self.graph = tf.Graph()  # 建立图
        with self.graph.as_default():  # 设为默认的graph
            self.keep_prob = tf.placeholder(tf.float32, None, name="keep_prob")
            self.train_inputs = tf.placeholder(tf.float32, shape=[None, None], name="train_inputs")  # 占位符，样本输入
            self.train_labels = tf.placeholder(tf.float32, shape=[None, None], name="train_labels")  # 占位符，样本标签
            # 使用GPU进行张量计算
            with tf.device('/cpu:0'):
                # 隐藏层1
                self.hidden_layer = self.layer(output_dim=self.hidden_dim[0], input_dim=self.input_dim,
                                               inputs=self.train_inputs,
                                               keep_prob=self.keep_prob,
                                               activation=None)
                # 隐藏层(>1)
                for num in range(len(self.hidden_dim) - 1):
                    self.hidden_layer = self.layer(output_dim=self.hidden_dim[num + 1], input_dim=self.hidden_dim[num],
                                                   inputs=self.hidden_layer,
                                                   keep_prob=self.keep_prob,
                                                   activation=None)
                # 输出层
                self.output_layer = self.layer(output_dim=self.output_dim, input_dim=self.hidden_dim[-1],
                                               inputs=self.hidden_layer,
                                               keep_prob=None,
                                               activation=None)

                # 最终使用sigmod函数输出预测结果
                self.prediction = tf.nn.sigmoid(self.output_layer)
                tf.identity(self.prediction, "output")

            # 使用sigmoid_cross_entropy作为损失函数
            self.loss = tf.reduce_mean(
                tf.nn.sigmoid_cross_entropy_with_logits(logits=self.output_layer, labels=self.train_labels))

            # 定义优化损失函数，使用Adam以及预设学习率训练参数
            self.optimizer = tf.train.AdamOptimizer(self.learning_rate).minimize(self.loss)
            self.init = tf.global_variables_initializer()

    def layer(self, output_dim, input_dim, inputs, keep_prob=None, activation=None):
        """
        神经网络层
        :param output_dim: 输出维度
        :param input_dim: 输入维度
        :param inputs: 输入数据
        :param activation: 采用激活函数
        :param keep_prob: drop_out保持率
        :return: 下一层输入
        """
        # 权重
        W = tf.Variable(tf.random_normal([input_dim, output_dim]))
        # 偏置
        b = tf.Variable(tf.random_normal([1, output_dim]))
        XWb = tf.matmul(inputs, W) + b
        if keep_prob is not None:
            XWb = tf.nn.dropout(XWb, keep_prob=keep_prob)
        if activation is None:
            outputs = XWb
        else:
            outputs = activation(XWb)
        return outputs

    def train(self, data_inputs, data_labels, keep_prob):
        """
        模型训练
        :param data_inputs: 样本输入
        :param data_labels: 样本标签
        :param keep_prob: dropout保留神经元概率
        """
        with tf.Session(graph=self.graph) as session:
            self.init.run()
            if not os.path.exists(self.save_model_path):
                os.mkdir(self.save_model_path)
            # 检查是否已存在模型，有则恢复训练，重新训练新模型需先删除模型
            if os.path.exists(self.save_model_path + "model.ckpt.meta"):
                ckpt = tf.train.latest_checkpoint(self.save_model_path)
                tf.train.Saver().restore(session, ckpt)
            total_loss = 0
            total_acc = 0
            max_acc = 0

            # 模型保存管理
            saver = tf.train.Saver(max_to_keep=2)
            start_time = time.time()
            # 使用tqdm.trange管理进度条输出
            with trange(self.num_steps) as t:
                for step in t:
                    t.set_description("Training")
                    batch_inputs, batch_labels, one_hot_label = next(
                        data_loader.generate_batch(data_inputs, data_labels, batch_size=self.batch_size))
                    feed_dict = {self.train_inputs: batch_inputs, self.train_labels: batch_labels,
                                 self.keep_prob: keep_prob}
                    _, loss, output = session.run([self.optimizer, self.loss, self.output_layer], feed_dict)
                    # 阈值判断输出
                    output = [[1 if i > self.threshold_value else 0 for i in j] for j in output]
                    batch_labels = list(batch_labels)
                    for num, value in enumerate(output):
                        if value == list(batch_labels[num]):
                            total_acc = total_acc + (1. / self.batch_size)
                    # 统计损失和准确率
                    total_loss = total_loss + loss

                    # 每steps次进行一次评估输出
                    if step % self.steps == 0 and step != 0:
                        # 计算平均损失
                        average_loss = total_loss / self.steps
                        average_acc = total_acc / self.steps
                        spend_time = time.time() - start_time
                        t.set_postfix(Loss="{:.9f}".format(average_loss), Accuracy="{:.9f}".format(average_acc))
                        self.fig_loss[self.count] = average_loss
                        self.fig_acc[self.count] = average_acc
                        if average_acc > self.save_model_threshold_value:
                            # 如果评估准确率历史最优，保存模型
                            self.save_model_threshold_value = average_acc
                            saver.save(sess=session, save_path=self.save_model_path + "model.ckpt")
                        # 每一次评估后损失与准确率置零
                        total_loss = 0
                        total_acc = 0
                        # 计数+1
                        self.count += 1
                        # 重置时间
                        start_time = time.time()

    def plot(self):
        # 绘制损失与准确率图
        fig, ax1 = plt.subplots()
        ax2 = ax1.twinx()
        lns1 = ax1.plot(np.arange(self.all_counts), self.fig_loss, label="Loss")
        lns2 = ax2.plot(np.arange(self.all_counts), self.fig_acc, 'r', label="Accuracy")
        ax1.set_xlabel('iteration')
        ax1.set_ylabel('training loss')
        ax2.set_ylabel('training accuracy')
        # 合并图例
        lns = lns1 + lns2
        labels = ["Loss", "Accuracy"]
        plt.legend(lns, labels, loc=7)
        plt.show()

    def main(self):
        """模型训练"""
        # 读取样本
        data_inputs, data_labels = data_loader.read_data(path=self.path, sheet_name=self.sheet_name, input_num=self.input_dim, label_num=self.output_dim)
        # 训练样本
        self.train(data_inputs, data_labels, self.keep_rate)
        # 绘制loss与acc图
        self.plot()

if __name__ == '__main__':
    # 创建类的实例
    ###########################################参数说明############################################
    # path: 数据输入路径，sheet_name：数据所在表单名， save_model_path: 模型保存路径
    # batch_size: 训练批次大小，learning_rate: 学习率, num_steps: 总迭代步数
    # model_dim为一个元祖，存放整个模型各层的维度[tuple:（输入层维度, 隐藏层1维度, 隐藏层2维度..., 输出层维度)]
    # steps: 每多少步进行评估，threshold_value: 输出结果阈值，dropout: 防过拟合，随机神经单元丢弃率
    # save_model_threshold_value: 模型保存阈值（评估准确率大于阈值保存模型）
    dnn = DNN(path="./data/样本.xlsx", sheet_name="Sheet2", save_model_path="./models/model-1/",
              batch_size=64, learning_rate=0.001, num_steps=20000, model_dim=(101, 64, 51, 40), steps=100,
              threshold_value=0.9, dropout=0.1, save_model_threshold_value=0.8)
    # 程序入口，开始进行训练
    dnn.main()


from pymongo import MongoClient
import requests
from watson_service import WatsonService
from data_service import DataService

def main():

    data_service = DataService()
    news_journals = data_service.get_known_online_news_journals()

    news_journal_articles = []
    for news_journal in news_journals:
        try:
            articles_of_journal_over_timeframe = data_service.get_articles_by_new_journal_over_timeframe(news_journal['url'])

        except Exception as query_failure:
            print('source name, source id, resulted in xmany articles or none at all within the timeframe')

    source_articles_by_source_id = {}
    for news_source in news_sources:
        articles = data_service.get_articles_from_online_news_journal_by_url(news_source['url'])
        source_articles_by_source_id[news_source['_id']] = articles
        if 'docs' in source_articles_by_source_id.keys():
            for article in source_articles_by_source_id['docs']['source']['enriched']['url']:
                pass

    # find the best possible news across one or more sources the most both singularly about a topic and with an
    # interesting or strong perspective
                # want news that is unique due to its paring of strange characters doing odd things, expressing a strong
                # and unique opinion towards common actions performed by a character in novel way

    # Hierarchy of boxes model, where the color of the lies of each rectange can convey different information, same as the
    # spacing between the rectangles

        stop_here = ""
    stop_here = ""

    query_url = 'https://topics.mediacloud.org/api/collections/9139458/stories'
    parameters = {
        'snapshotId': '1477',
        'timespandId': '39849',
        'sort': 'inlink',
        'limit': '5000'
    }
    # response = requests.get(query_url, params=parameters)
    # stop_here = ""

    client = MongoClient('localhost', 27017)
    database = client['AkashicRecords']
    articles = database['historic_political_article_data']
    cbt = database['cleaned_breitbart_test']

    articles_found = []
    for article in articles.find():
        articles_found.append(article)

    cleaned_articles = []
    for article in articles_found:
        if len(article['entities']) > 5:
            article['entities'] = article['entities'][:5]

            stop_here = ""

    for article in articles_found:
        relation_by_sent_id = {}
        for relation in article['relations']:
            try:
                sent_id = hash(relation['sentence'])
                if 'subject' in relation.keys():
                    if 'object' in relation.keys():
                        pass
                if 'object' in relation.keys():
                    pass
            except Exception as e:
                print(e.args)

    # watson_service = WatsonService()

    # articles_found = data_service.pull_from_source('http://www.breitbart.com')
    cleaned_articles = data_service.clean_source_articles('breitbart.com', articles_found)
    cbt.insert_many(cleaned_articles)

    article_data_list = []
    for article in articles.find():
        article_element_types = []
        relevance_threshold = 0.80

        for entity in article['entities']:
            if entity['relevance'] > relevance_threshold:
                if 'knowledgeGraph' in entity.keys():
                    if 'typeHierarchy' in entity['knowledgeGraph'].keys():
                        article_element_types.append(entity['knowledgeGraph']['typeHierarchy'].split('/')[1:])

        for keyword in article['keywords']:
            if keyword['relevance'] > relevance_threshold:
                if 'knowledgeGraph' in keyword.keys():
                    if 'typeHierarchy' in keyword['knowledgeGraph'].keys():
                        article_element_types.append(keyword['knowledgeGraph']['typeHierarchy'].split('/')[1:])

        article_data_list.append(article_element_types)

    stop_here = ""

if __name__ == '__main__':
    main()


from collections import deque
n=int(input())
sea=[list(map(int,input().split())) for _ in range(n)]

d=[(1,0),(-1,0),(0,1),(0,-1)]

def bfs(q,shark):
    global cnt
    checklist=[]
    checkpoint=n**2
    while q:
        x,y,t=q.popleft()
        if t>checkpoint:
            sea[shark[0]][shark[1]]=0
            checklist.sort()
            sea[checklist[0][0]][checklist[0][1]]=9
            cnt+=1
            return t
        for dx,dy in d:
            nx=x+dx
            ny=y+dy
            if 0<=nx<n and 0<=ny<n and not v[nx][ny]:
                if sea[nx][ny]==0 or sea[nx][ny]==size:
                    v[nx][ny]=True
                    q.append((nx,ny,t+1))
                elif sea[nx][ny]<size:
                    checkpoint=t
                    checklist.append((nx,ny))

    if checklist:
        sea[shark[0]][shark[1]]=0
        checklist.sort()
        sea[checklist[0][0]][checklist[0][1]]=9
        cnt+=1
        return t+1

    return False
cnt=0
size=2
ans=0
while True:
    v=[[False]*n for _ in range(n)]
    for i in range(n):
        for j in range(n):
            if sea[i][j]==9:
                q=deque([(i,j,0)])
                v[i][j]=True
                shark=(i,j)
    tmp=bfs(q,shark)
    if not tmp:
        break
    ans+=tmp
    if cnt==size:
        size+=1
        cnt=0

print(ans)


from flask import Flask
from app.extensions import db, migrate, login_manager
from app.user import user

def create_app():
    app = Flask(__name__)
    app.config.from_pyfile('config.py')

    db.init_app(app)
    migrate.init_app(app, db)
    login_manager.init_app(app)

    app.register_blueprint(user, url_prefix='/user')

    return app


from flask import Flask, jsonify, request, render_template
from datetime import datetime
from db import despesas, tipos_de_pagamento, categorias, Pagamento, Categoria

app = Flask(__name__)

# criar os tipos de pagamento // importada do banco de dados
# criar as categorias // importada do banco de dados

# pagina inicial
@app.route('/', methods=['GET'])
def pagina_inicial():
    return render_template('index.html')

# listar todas as despesas do mes vigente
@app.route('/api/despesas', methods=['GET'])
def get_despesas():
    lista = []
    total = 0
    for despesa in despesas:
        if despesa.get('data').month == datetime.today().month:
            lista.append(despesa)
            total += despesa.get('valor')
    return jsonify({'data': lista, 'total': total, 'success': True})

# listar todas as depesas totais
@app.route('/api/despesasTotais', methods=['GET'])
def get_todas_despesas():
    return jsonify({'data':despesas, 'success': True})

# listas despesas por categoria
@app.route('/api/despesas/<categoria>', methods=['GET'])
def get_por_categoria(categoria):
    lista = []
    for despesa in despesas:
        if despesa.get('categoria') == categoria:
            lista.append(despesa)
    return jsonify({'data': lista, 'success': True})

# adicionar despesa
@app.route('/api/add/despesa', methods=['POST'])
def cadastrar_despesa():
    despesa = request.get_json()

    id = despesa['id'] # get id
    valor = despesa['valor'] # get valor
    descricao = despesa['descricao'] # get descricao

    for index, pagamento in enumerate(tipos_de_pagamento): # converte o pagamento
        if despesa['pagamento'] == pagamento.nome:
            pagamento = tipos_de_pagamento[index]

    for index, categoria in enumerate(categorias): # converte a categoria
        if despesa['categoria'] == categoria.nome:
            categoria = categorias[index]

    data = datetime.strptime(despesa['data'], '%d/%m/%y').date() # converte em data

    nova_despesa = {
        'id': id,
        'valor': valor,
        'descricao': descricao,
        'pagamento': pagamento.nome,
        'categoria': categoria.nome,
        'data': data
    }

    despesas.append(nova_despesa)

    return jsonify({'data': despesas, 'success': True})

# adicionar tipo de pagamento
@app.route('/api/add/pagamento', methods=['POST'])
def cadastrar_pagamento():
    pagamento = request.get_json()

    id = pagamento['id']
    nome = pagamento['nome']

    novo_pagamento = Pagamento(id, nome)

    tipos_de_pagamento.append(novo_pagamento)

    return jsonify({'data': f'Pagamento adicionado: {novo_pagamento.id} - {novo_pagamento.nome}',
                    'success': True})

# adicionar nova categoria de compra
@app.route('/api/add/categoria', methods=['POST'])
def cadastrar_categoria():
    categoria = request.get_json()

    id = categoria['id']
    nome = categoria['nome']
    descricao = categoria['descricao']

    nova_categoria = Categoria(id, nome, descricao)

    categorias.append(nova_categoria)

    return jsonify({'data': f'Categoria adicionada: {nova_categoria.id} - {nova_categoria.nome} - {nova_categoria.descricao}',
                   'success': True})

# listar tipos de pagamentos
@app.route('/api/pagamentos', methods=['GET'])
def mostrar_pagamentos():
    lista = []
    for value in tipos_de_pagamento:
        lista.append({'id': value.id,
                      'nome': value.nome})

    return jsonify({'data': lista, 'success': True})

# listar categorias e suas descricoes
@app.route('/api/categorias', methods=['GET'])
def mostrar_categorias():
    lista = []
    for value in categorias:
        lista.append({'id': value.id,
                      'nome': value.nome,
                      'descricao': value.descricao})

    return jsonify({'data': lista, 'success': True})

app.run(port=5000, host='localhost', debug=True)


import pandas as pd
from tqdm import tqdm
import utilities as ut
import matplotlib.pyplot as plt
from sklearn.neural_network import MLPClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import cross_val_score
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import f1_score, recall_score, plot_confusion_matrix, roc_auc_score, classification_report,\
    accuracy_score, precision_score

if __name__ == '__main__':

    # =================================================================================
    # ==========================load prognostics data==================================
    data_path = r"/ML2/project/extra_cell_lymphoma_dev/prognostics_dev.xlsx"  # enter path
    df = ut.load_data_and_transform(data_path, "prognostics")

    # =================================================================================
    # ===========================data exploration&label handling=======================
    # we saw that their are patient w/o label.
    # in addition we saw that there is corr between missing labels and na in more than 4 features.
    # thus we decided to delete rows w/o labels.
    print(df.isnull().sum(axis=0))
    print(df.describe())
    for i in range(2,27,9): # features distribution
        ut.draw_multiple_histograms(df[df.columns[i:i+9]], 2, 2)
    ut.heatmap(df)
    ut.split_pos_and_neg(df)
    # we decided to delete number of features according to their distributions:
    # 1. Diagnosis - obviously all the patients has the same diagnosis so this is redundant
    # 2. Treatment - half of the data is missing and the other half has the same treatment.
    # 3. Biopsy Type - the major of the feature is pre-treatment
    # these features from the future thus can't be part of the model
    # 1. Included in survival analysis
    # 2. all progressions
    # 3. follow ups
    curr_df = ut.preprocessing_dataframe(df)
    print(curr_df.describe())
    # ==================================================================================================================
    # ============================= baseline model imputed missing values by traditional way ===========================
    baseline_df_trad = curr_df.copy()
    baseline_df_trad = ut.fill_missing_values_traditional_way(baseline_df_trad)
    baseline_df_trad = baseline_df_trad.round(
        {'Age': 0, 'IPI Group': 0, 'Ann Arbor Stage': 0, 'ECOG Performance Status': 0})
    X_train_trad, X_val_trad, y_train_trad, y_val_trad = ut.train_test_split_and_rename(baseline_df_trad,
                                                                                baseline_df_trad[
                                                                                'Number of Extranodal Sites'
                                                                                ])

    rf0 = RandomForestClassifier()
    log_r0 = LogisticRegression()
    knn0 = KNeighborsClassifier()
    mlp0 = MLPClassifier()
    models = [rf0, log_r0, knn0, mlp0]
    for clf in models:
        mean_auc = ut.cross_validation(10,X_train_trad,y_train_trad,clf)
        if clf == rf0:
            baseline_auc = mean_auc

    # ==================================================================================================================
    # =============================== baseline model with missing values imputed by KNN ================================
    curr_df = ut.fill_missing_values_by_KNN(curr_df)
    curr_df = curr_df.round({'Age': 0, 'IPI Group': 0, 'Ann Arbor Stage': 0, 'ECOG Performance Status': 0})
    # split train labels and validation
    X_train, X_val, y_train, y_val = ut.train_test_split_and_rename(curr_df, curr_df['Number of Extranodal Sites'])
    rf = RandomForestClassifier()
    log_r = LogisticRegression()
    knn = KNeighborsClassifier()
    mlp = MLPClassifier()
    models = [rf, log_r, knn, mlp]
    for clf in models:
        ut.cross_validation(10,X_train,y_train,clf)
    # =====================================================================================================
    # ============================================grid_search==============================================
    # rf = RandomForestClassifier()
    # rf_param_grid = {
    #      'max_depth': [10,20,50,80,110],
    #      'max_features': ['auto', 'sqrt'],
    #      'min_samples_leaf': [1,2,6,10],
    #      'min_samples_split': [2,6,10],
    #      'n_estimators': [10,100,600,1000]}
    # train_size = 0.8
    # # In the first step we will split the data in training and remaining dataset
    # X_train_grid, X_rem, y_train_grid, y_rem = X_train_trad, X_val_trad, y_train_trad, y_val_trad
    #
    # # Now since we want the valid and test size to be equal (10% each of overall data).
    # # we have to define valid_size=0.5 (that is 50% of remaining data)
    # test_size = 0.5
    # X_val_grid, X_test_grid, y_val_grid, y_test_grid = ut.train_test_split(X_rem, y_rem, test_size=0.5)
    #
    # best_param = ut.run_grid_search(rf, rf_param_grid, X_train_grid, y_train_grid, X_val_grid, y_val_grid)
    #
    # # rf_grid = RandomForestClassifier(**best_param)
    # # rf_grid.fit(X_train_grid,y_train_grid)
    # # plot_roc_curve(rf_grid, X_test_grid,y_test_grid)
    # # plt.show()
    # ==========================================================================================================
    # the two Imputing methods yield the same results, thus we will continue with the traditional baseline
    # In addition, after gridsearch we found the best parameters are the default

    X_train, X_val, y_train, y_val = X_train_trad, X_val_trad, y_train_trad, y_val_trad
    # ==========================================================================================================
    # ===================================load genetic data======================================================
    data_path = r"/ML2/project/extra_cell_lymphoma_dev/genomics_dev.xlsx"  # enter  path
    genomic_df = ut.load_data_and_transform(data_path, "genomics")
    X_train, y_train, X_val, y_val = ut.join_genomics(genomic_df, X_train, y_train, X_val, y_val)
    # ==========================================================================================================
    # ===================================drop features by domain knowledge =====================================
    X_train = X_train.drop(['ECOG Performance Status', 'IPI Range', 'dbGaP_accession_phs000178',
                            'dbGaP_accession_phs001444', 'Gene_Expression_Subgroup_ABC', 'Gene_Expression_Subgroup_GCB',
                            'Gene_Expression_Subgroup_Unclass', 'Genetic_Subtype_BN2', 'Genetic_Subtype_EZB',
                            'Genetic_Subtype_MCD', 'Genetic_Subtype_N1', 'Genetic_Subtype_Other'], axis=1)
    X_val = X_val.drop(['ECOG Performance Status', 'IPI Range', 'dbGaP_accession_phs000178',
                        'dbGaP_accession_phs001444', 'Gene_Expression_Subgroup_ABC', 'Gene_Expression_Subgroup_GCB',
                        'Gene_Expression_Subgroup_Unclass', 'Genetic_Subtype_BN2', 'Genetic_Subtype_EZB',
                        'Genetic_Subtype_MCD', 'Genetic_Subtype_N1', 'Genetic_Subtype_Other'], axis=1)
    # =========================================================================================================
    # ===================================run basic model with genomic data=====================================
    print("=================run basic model with all genomic data=================")
    rf2 = RandomForestClassifier()
    auc = ut.run_model_and_draw_ROC(rf2, X_train, y_train, X_val, y_val)
    print(f"for the baseline model with all the genomic data the AUC is: {auc}")

    # =========================================================================================================
    # ===================================we will try reducing dimension by using pca===========================
    print("=================run model with reduced dimensions by PCA=================")
    X_train_pca, X_val_pca = ut.get_reduced_dimensions_df(X_train, X_val)
    y_train_pca = y_train.reindex(X_train_pca.index)
    y_val_pca = y_val.reindex(X_val_pca.index)
    print(f"Reduced dimensions to: {X_train_pca.shape}")
    rf3 = RandomForestClassifier()
    auc = ut.run_model_and_draw_ROC(rf3, X_train_pca, y_train_pca, X_val_pca, y_val_pca)
    print(f"for the model with reduced dimensions by PCA the AUC is: {auc}")

    # =========================================================================================================
    # ===================================We will try feature selection by Domain knowledge====================
    print("=================run model with features selected by Domain knowledge=================")
    relevant_features = ['Gender', 'Age', 'Ann Arbor Stage', 'LDH Ratio',
                         'IPI Group', 'JAM2', 'JAM3', 'MYC', 'POU2AF1', 'EZH2',
                         'BCL2', 'AQP9',
                         'LMBR1L', 'FGF20', 'TANK', 'CRP', 'ORM1', 'JAK1', 'BACH1', 'MTCP1', 'IFITM1', 'TNFSF10',
                         'FGF12', 'RFX5', 'LAP3']
    print(f"The selected features are:{relevant_features}")
    X_train_fs = X_train[relevant_features]
    X_val_fs = X_val[relevant_features]

    rf4 = RandomForestClassifier()
    auc = ut.run_model_and_draw_ROC(rf4, X_train_fs, y_train, X_val_fs, y_val)
    print(f"for the model with features selected by Domain knowledge the AUC is: {auc}")

    # =========================================================================================================
    # =================================== lets try feature selection using forward selection ==================
    def forward_selection(data, target, X_val, y_val,auc, significance_level=0.01):
        initial_features = data.columns.tolist()
        best_features = ['Gender', 'Age', 'Ann Arbor Stage', 'LDH Ratio',
                         'IPI Group']
        scores = [auc,auc,auc]
        best_features_and_scores = {}
        while (len(initial_features) > 0):
            print(best_features_and_scores)
            remaining_features = list(set(initial_features) - set(best_features))
            new_pval = pd.Series(index=remaining_features)
            for new_column in tqdm(remaining_features):
                new_pval[new_column] = 0
                for i in range(3):
                    model = RandomForestClassifier().fit(data[best_features + [new_column]], target)
                    new_pval[new_column] += roc_auc_score(y_val,
                                                          model.predict_proba(X_val[best_features + [new_column]])[:,
                                                          1])
                new_pval[new_column] = new_pval[new_column] / 3
                print(f"{new_column}:", new_pval[new_column])
            max_score = new_pval.max()
            if (max_score - scores[-1] > significance_level):
                best_features.append(new_pval.idxmax())
                scores.append(max_score)
                best_features_and_scores[new_pval.idxmax()] = max_score
            else:
                best_features.append(new_pval.idxmax())
                scores.append(scores[-1])
                if scores[-1] == scores[-2] and scores[-1] == scores[-3]:
                    best_features = best_features[:len(best_features) - 2]
                    print(f"scores: {scores}")
                    break
        return best_features, best_features_and_scores

    # =========================================================================================================
    # ===================3 experiments for the feature selection===============================================
    for experiment in range(1,4):
        best_features, best_features_and_scores = forward_selection(X_train, y_train, X_val, y_val, baseline_auc)
        print(f"best features : {best_features}")
        print(f"best features for exp. {experiment}: {best_features_and_scores}")
        ut.open_file_and_save(f"exp.{experiment}", best_features_and_scores)

    # after finding the best features plot roc curves that compares the baseline model to the one
    # with the best genetics features

    print("=================run model with features selected by Forward Selection=================")

    print(f"best features: {best_features}")
    rf_basic = RandomForestClassifier()
    rf_genetic = RandomForestClassifier()
    models = [rf_basic, rf_genetic]
    relevant_features = {rf_basic: ['Gender', 'Age', 'Ann Arbor Stage', 'LDH Ratio', 'IPI Group'],
                         rf_genetic: best_features}
    ut.plot_multiple_ROC_curves(X_train, y_train, X_val, y_val, models, relevant_features)
    X_train = X_train[best_features]
    X_val = X_val[best_features]

    # =========================================================================================================
    # =====================================CM and scores=============================================
    rf = RandomForestClassifier()
    rf.fit(X_train, y_train)
    plot_confusion_matrix(rf, X_val, y_val)
    plt.show()
    y_pred_test = rf.predict(X_val)
    # accuracy: (tp + tn) / (p + n)
    accuracy = accuracy_score(y_val, y_pred_test)
    print(f"Accuracy: {accuracy}")
    # precision tp / (tp + fp)
    precision = precision_score(y_val, y_pred_test)
    print(f"Precision: {precision}")
    # recall: tp / (tp + fn)
    recall = recall_score(y_val, y_pred_test)
    print(f"Recall: {recall}")
    # f1: 2 tp / (2 tp + fp + fn)
    f1 = f1_score(y_val, y_pred_test)
    print(f"F1 score: {f1}")
    # View the classification report for test data and predictions
    print(classification_report(y_val, y_pred_test))
    ut.plot_train_test_ROC_curves(X_train, y_train, X_val, y_val, rf)


from mxnet.gluon.data import dataset, DataLoader
import numpy as np
import mxnet.ndarray as ndarray
from utils.augmentations import *

class MyTransform:
    def __init__(self, im, opts):
        if opts.is_val:
            transforms_list_idx = [0, 14, 1]
        else:
            transforms_list_idx = [0, ] + [opts.transforms_list_idx[j]
                                           for j in np.random.choice(opts.transforms_list_idx.__len__(),
                                                                     opts.transforms_list_idx.__len__(), replace=False)] \
                                  + [14, 1, ]
        self.to_norm = not opts.already_normed
        self.norm_thr = opts.norm_thr
        self.nsee = opts.num_slices_each_end
        self.label_in_last_channel = opts.label_in_last_channel
        self.max_angle = opts.max_angle
        self.coor_maps = opts.coor_maps
        self.is_val = opts.is_val
        self.zdim = opts.zdim
        self.mid_len_ratio = opts.mid_len_ratio
        self.translate_ratio = opts.translate_ratio
        self.mean_scaling = opts.mean_scaling
        self.crop_size_list = opts.crop_size_list
        self.crop_size = opts.crop_size
        self.use_ADC = opts.use_ADC
        self.cov = opts.center_translation_cov
        self.transforms_dict = {
            'sample_slices': sample_slices,  # 0
            'concat_coor_maps': concat_coor_maps,  # concat_coor_maps index must be 0  # 1
            'rand_scale': rand_scale,  # 2
            'drop_out': drop_out,  # 3
            'contrast_norm': contrast_norm,  # 4
            'deform': deform,  # 5
            'modify_values': modify_values,  # 6
            'Gauss_blur': Gauss_blur,  # 7
            'add_Gauss_noise': add_noise,  # 8
            'rand_translate': rand_translate,  # 9
            'rand_rot': rand_rot,  # 10
            'rand_fliplr': rand_fliplr,  # 11
            'center_crop_and_scale': None,  # 12
            'swap_channels': swap_channels,  # 13
            'crop_and_resize': crop_and_resize,  # 14
            'rand_flipud': rand_flipud, # 15
        }
        self.transforms_list = list(self.transforms_dict.keys())
        self.transforms_list = [self.transforms_list[i] for i in transforms_list_idx]
        self.im = im

    def random_transform(self):
        for fn in self.transforms_list:
            self.im = self.transforms_dict[fn](self)
        return self.im

class MyDataset(dataset.Dataset):
    """A dataset that combines multiple dataset-like objects, e.g.
    Datasets, lists, arrays, etc.

    The i-th sample is defined as `(x1[i], x2[i], ...)`.

    Parameters
    ----------
    *args : one or more dataset-like objects
        The data arrays.
    """
    def __init__(self, opts, *args):
        assert len(args) > 0, "Needs at least 1 arrays"
        self._length = len(args[0])
        self._data = []
        self.opts = opts
        for i, data in enumerate(args):
            assert len(data) == self._length, \
                "All arrays must have the same length; array[0] has length %d " \
                "while array[%d] has %d." % (self._length, i+1, len(data))
            if isinstance(data, ndarray.NDArray) and len(data.shape) == 1:
                data = data.asnumpy()
            self._data.append(data)

    def __getitem__(self, idx):
        if len(self._data) == 1:
            return MyTransform(self._data[0][idx], self.opts).random_transform()
        else:
            return tuple(MyTransform(data[idx], self.opts).random_transform() for data in self._data)

    def __len__(self):
        return self._length

if __name__ == "__main__":
    x = np.zeros(shape=(5, 2, 10, 30, 30))

    d = MyDataset(x)
    data_loader = DataLoader(d, batch_size=2)

    for b in data_loader:
        print(b[0].shape)


import cv2

# haarcascade 불러오기
face_cascade = cv2.CascadeClassifier(cv2.data.haarcascades + 'haarcascade_frontalface_default.xml')
eye_cascade = cv2.CascadeClassifier(cv2.data.haarcascades + 'haarcascade_eye.xml')

# 이미지 불러오기
original_num = 2
img = cv2.imread(str(original_num)+'.jpg')
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

# 얼굴인식 및 인식개수가 1개 초과 시 종료(나중에 반복문으로 리턴 구현)
faces = face_cascade.detectMultiScale(gray, 1.3, 5)

if (len(faces)==1):
    for (x, y, w, h) in faces:
        cropped = img[y - int(h / 4):y + h + int(h / 1), x - int(w / 4):x + w + int(w / 4)]  # 상:하, 좌:우
else:
    print("얼굴 다중인식")
    exit()

# 자른 사진 크기 조정 #게임 IF 이미지 크기: 400, 500
if cropped.shape[1]*cropped.shape[0]>2000:
    resize_img = cv2.resize(img, dsize=(400, 500), interpolation=cv2.INTER_AREA)
else:
    resize_img = cv2.resize(img, dsize=(400, 500), interpolation=cv2.INTER_CUBIC)

# 자른 사진 저장
change_num = 99
cv2.imwrite(str(change_num)+".jpg",cropped)

# 원본 사진, 자른 사진 출력
cv2.imshow('image', img)
cv2.imshow("cropped", cropped)

key = cv2.waitKey(0)
cv2.destroyAllWindows()


import logging
from metadrive.constants import Semantics
import math
from typing import List, Dict

from panda3d.bullet import BulletBoxShape, BulletGhostNode
from panda3d.core import Vec3, LQuaternionf, Vec4, TextureStage, RigidBodyCombiner, \
    SamplerState, NodePath, Texture, Material

from metadrive.base_class.base_object import BaseObject
from metadrive.component.lane.abs_lane import AbstractLane
from metadrive.component.lane.point_lane import PointLane
from metadrive.component.road_network.node_road_network import NodeRoadNetwork
from metadrive.component.road_network.road import Road
from metadrive.constants import MetaDriveType, CamMask, PGLineType, PGLineColor, DrivableAreaProperty
from metadrive.engine.asset_loader import AssetLoader
from metadrive.engine.core.physics_world import PhysicsWorld
from metadrive.utils.coordinates_shift import panda_vector, panda_heading
from metadrive.utils.math import norm

logger = logging.getLogger(__name__)

class BaseBlock(BaseObject, DrivableAreaProperty):
    """
    Block is a driving area consisting of several roads
    Note: overriding the _sample() function to fill block_network/respawn_roads in subclass
    Call Block.construct_block() to add it to world
    """
    ID = "B"

    def __init__(
        self, block_index: int, global_network: NodeRoadNetwork, random_seed, ignore_intersection_checking=False
    ):
        super(BaseBlock, self).__init__(str(block_index) + self.ID, random_seed, escape_random_seed_assertion=True)
        # block information
        assert self.ID is not None, "Each Block must has its unique ID When define Block"
        assert len(self.ID) == 1, "Block ID must be a character "

        self.block_index = block_index
        self.ignore_intersection_checking = ignore_intersection_checking

        # each block contains its own road network and a global network
        self._global_network = global_network
        self.block_network = self.block_network_type()

        # a bounding box used to improve efficiency x_min, x_max, y_min, y_max
        self._bounding_box = None

        # used to spawn npc
        self._respawn_roads = []
        self._block_objects = None

        if self.render and not self.use_render_pipeline:
            self.ts_color = TextureStage("color")
            self.ts_normal = TextureStage("normal")
            self.ts_normal.setMode(TextureStage.M_normal)

            # Only maintain one copy of asset
            self.road_texture = self.loader.loadTexture(AssetLoader.file_path("textures", "sci", "new_color.png"))
            self.road_normal = self.loader.loadTexture(AssetLoader.file_path("textures", "sci", "normal.jpg"))
            self.road_texture.set_format(Texture.F_srgb)
            self.road_normal.set_format(Texture.F_srgb)
            self.road_texture.setMinfilter(SamplerState.FT_linear_mipmap_linear)
            self.road_texture.setAnisotropicDegree(8)

            # # continuous line
            # self.lane_line_model = self.loader.loadModel(AssetLoader.file_path("models", "box.bam"))
            # self.lane_line_model.setPos(0, 0, -DrivableAreaProperty.LANE_LINE_GHOST_HEIGHT / 2)
            self.lane_line_texture = self.loader.loadTexture(AssetLoader.file_path("textures", "sci", "floor.jpg"))
            # self.lane_line_model.setScale(DrivableAreaProperty.STRIPE_LENGTH*4,
            #                                    DrivableAreaProperty.LANE_LINE_WIDTH,
            #                                    DrivableAreaProperty.LANE_LINE_THICKNESS)
            # # self.lane_line_normal = self.loader.loadTexture(
            # #     AssetLoader.file_path("textures", "sci", "floor_normal.jpg"))
            # # self.lane_line_texture.set_format(Texture.F_srgb)
            # # self.lane_line_normal.set_format(Texture.F_srgb)
            # self.lane_line_model.setTexture(self.ts_color, self.lane_line_texture)
            # # self.lane_line_model.setTexture(self.ts_normal, self.lane_line_normal)
            #
            # # # broken line
            # self.broken_lane_line_model = self.loader.loadModel(AssetLoader.file_path("models", "box.bam"))
            # self.broken_lane_line_model.setScale(DrivableAreaProperty.STRIPE_LENGTH,
            #                                           DrivableAreaProperty.LANE_LINE_WIDTH,
            #                                           DrivableAreaProperty.LANE_LINE_THICKNESS)
            # self.broken_lane_line_model.setPos(0, 0, -DrivableAreaProperty.LANE_LINE_GHOST_HEIGHT / 2)
            # self.broken_lane_line_model.setTexture(self.ts_color, self.lane_line_texture)

            # side
            self.side_texture = self.loader.loadTexture(AssetLoader.file_path("textures", "sidewalk", "color.png"))
            self.side_texture.set_format(Texture.F_srgb)
            self.side_texture.setMinfilter(SamplerState.FT_linear_mipmap_linear)
            self.side_texture.setAnisotropicDegree(8)
            self.side_normal = self.loader.loadTexture(AssetLoader.file_path("textures", "sidewalk", "normal.png"))
            self.side_normal.set_format(Texture.F_srgb)
            self.sidewalk = self.loader.loadModel(AssetLoader.file_path("models", "box.bam"))
            self.sidewalk.setTwoSided(False)
            self.sidewalk.setTexture(self.ts_color, self.side_texture)
            # self.sidewalk = self.loader.loadModel(AssetLoader.file_path("models", "output.egg"))
            # self.sidewalk.setTexture(self.ts_normal, self.side_normal)

    def _sample_topology(self) -> bool:
        """
        Sample a new topology to fill self.block_network
        """
        raise NotImplementedError

    def construct_block(
        self,
        root_render_np: NodePath,
        physics_world: PhysicsWorld,
        extra_config: Dict = None,
        no_same_node=True,
        attach_to_world=True
    ) -> bool:
        """
        Randomly Construct a block, if overlap return False
        """
        self.sample_parameters()

        if not isinstance(self.origin, NodePath):
            self.origin = NodePath(self.name)
        # else:
        #     print("Origin already exists: ", self.origin)

        self._block_objects = []
        if extra_config:
            assert set(extra_config.keys()).issubset(self.PARAMETER_SPACE.parameters), \
                "Make sure the parameters' name are as same as what defined in pg_space.py"
            raw_config = self.get_config(copy=True)
            raw_config.update(extra_config)
            self.update_config(raw_config)
        self._clear_topology()
        success = self._sample_topology()
        self._global_network.add(self.block_network, no_same_node)

        self._create_in_world()
        self.attach_to_world(root_render_np, physics_world)

        if not attach_to_world:
            self.detach_from_world(physics_world)

        return success

    def destruct_block(self, physics_world: PhysicsWorld):
        self._clear_topology()
        self.detach_from_world(physics_world)

        self.origin.removeNode()
        self.origin = None

        self.dynamic_nodes.clear()
        self.static_nodes.clear()
        for obj in self._block_objects:
            obj.destroy()
        self._block_objects = None

    def construct_from_config(self, config: Dict, root_render_np: NodePath, physics_world: PhysicsWorld):
        success = self.construct_block(root_render_np, physics_world, config)
        return success

    def get_respawn_roads(self):
        return self._respawn_roads

    def get_respawn_lanes(self):
        """
        return a 2-dim array [[]] to keep the lane index
        """
        ret = []
        for road in self._respawn_roads:
            lanes = road.get_lanes(self.block_network)
            ret.append(lanes)
        return ret

    def get_intermediate_spawn_lanes(self):
        """Return all lanes that can be used to generate spawn intermediate vehicles."""
        raise NotImplementedError()

    def _add_one_respawn_road(self, respawn_road: Road):
        assert isinstance(respawn_road, Road), "Spawn roads list only accept Road Type"
        self._respawn_roads.append(respawn_road)

    def _clear_topology(self):
        if len(self._global_network.graph.keys()) > 0:
            self._global_network -= self.block_network
        self.block_network.graph.clear()
        self.PART_IDX = 0
        self.ROAD_IDX = 0
        self._respawn_roads.clear()

    """------------------------------------- For Render and Physics Calculation ---------------------------------- """

    def _create_in_world(self, skip=False):
        """
        Create NodePath and Geom node to perform both collision detection and render

        Note: Override the create_in_world() function instead of this one, since this method severing as a wrapper to
        help improve efficiency
        """
        self.lane_line_node_path = NodePath(RigidBodyCombiner(self.name + "_lane_line"))
        self.sidewalk_node_path = NodePath(RigidBodyCombiner(self.name + "_sidewalk"))
        self.lane_node_path = NodePath(RigidBodyCombiner(self.name + "_lane"))
        self.lane_vis_node_path = NodePath(RigidBodyCombiner(self.name + "_lane_vis"))

        self.sidewalk_node_path.setTag("type", Semantics.SIDEWALK.label)
        self.lane_vis_node_path.setTag("type", Semantics.ROAD.label)
        self.lane_line_node_path.setTag("type", Semantics.LANE_LINE.label)

        if skip:  # for debug
            pass
        else:
            self.create_in_world()

        self.lane_line_node_path.flattenStrong()
        self.lane_line_node_path.node().collect()

        self.sidewalk_node_path.flattenStrong()
        self.sidewalk_node_path.node().collect()
        self.sidewalk_node_path.hide(CamMask.ScreenshotCam)

        # only bodies reparent to this node
        self.lane_node_path.flattenStrong()
        self.lane_node_path.node().collect()

        self.lane_vis_node_path.flattenStrong()
        self.lane_vis_node_path.node().collect()
        self.lane_vis_node_path.hide(CamMask.DepthCam | CamMask.ScreenshotCam | CamMask.SemanticCam)

        self.origin.hide(CamMask.Shadow)

        self.sidewalk_node_path.reparentTo(self.origin)
        self.lane_line_node_path.reparentTo(self.origin)
        self.lane_node_path.reparentTo(self.origin)
        self.lane_vis_node_path.reparentTo(self.origin)
        try:
            self._bounding_box = self.block_network.get_bounding_box()
        except:
            if len(self.block_network.graph) > 0:
                logging.warning("Can not find bounding box for it")
            self._bounding_box = None, None, None, None

        self._node_path_list.append(self.sidewalk_node_path)
        self._node_path_list.append(self.lane_line_node_path)
        self._node_path_list.append(self.lane_node_path)
        self._node_path_list.append(self.lane_vis_node_path)

    def create_in_world(self):
        """
        Create lane in the panda3D world
        """
        raise NotImplementedError

    def add_body(self, physics_body):
        raise DeprecationWarning(
            "Different from common objects like vehicle/traffic sign, Block has several bodies!"
            "Therefore, you should create BulletBody and then add them to self.dynamics_nodes "
            "manually. See in construct() method"
        )

    def get_state(self) -> Dict:
        """
        The record of Block type is not same as other objects
        """
        return {}

    def set_state(self, state: Dict):
        """
        Block type can not set state currently
        """
        pass

    def _add_lane_line(self, lane: AbstractLane, colors: List[Vec4], contruct_two_side=True):
        raise DeprecationWarning("Leave for argoverse using")
        if isinstance(lane, PointLane):
            parent_np = self.lane_line_node_path
            lane_width = lane.width_at(0)
            for c, i in enumerate([-1, 1]):
                line_color = colors[c]
                acc_length = 0
                if lane.line_types[c] == PGLineType.CONTINUOUS:
                    for segment in lane.segment_property:
                        lane_start = lane.position(acc_length, i * lane_width / 2)
                        acc_length += segment["length"]
                        lane_end = lane.position(acc_length, i * lane_width / 2)
                        middle = (lane_start + lane_end) / 2
                        self._add_lane_line2bullet(
                            lane_start, lane_end, middle, parent_np, line_color, lane.line_types[c]
                        )

    def _add_box_body(self, lane_start, lane_end, middle, parent_np: NodePath, line_type, line_color):
        raise DeprecationWarning("Useless, currently")
        length = norm(lane_end[0] - lane_start[0], lane_end[1] - lane_start[1])
        if PGLineType.prohibit(line_type):
            node_name = MetaDriveType.LINE_SOLID_SINGLE_WHITE if line_color == PGLineColor.GREY else MetaDriveType.LINE_SOLID_SINGLE_YELLOW
        else:
            node_name = MetaDriveType.BROKEN_LINE
        body_node = BulletGhostNode(node_name)
        body_node.set_active(False)
        body_node.setKinematic(False)
        body_node.setStatic(True)
        body_np = parent_np.attachNewNode(body_node)

        self._node_path_list.append(body_np)

        shape = BulletBoxShape(
            Vec3(length / 2, DrivableAreaProperty.LANE_LINE_WIDTH / 2, DrivableAreaProperty.LANE_LINE_GHOST_HEIGHT)
        )
        body_np.node().addShape(shape)
        mask = DrivableAreaProperty.CONTINUOUS_COLLISION_MASK if line_type != PGLineType.BROKEN else DrivableAreaProperty.BROKEN_COLLISION_MASK
        body_np.node().setIntoCollideMask(mask)
        self.static_nodes.append(body_np.node())

        body_np.setPos(panda_vector(middle, DrivableAreaProperty.LANE_LINE_GHOST_HEIGHT / 2))
        direction_v = lane_end - lane_start
        # theta = -numpy.arctan2(direction_v[1], direction_v[0])
        theta = panda_heading(math.atan2(direction_v[1], direction_v[0]))

        body_np.setQuat(LQuaternionf(math.cos(theta / 2), 0, 0, math.sin(theta / 2)))

    @property
    def block_network_type(self):
        """
        There are two type of road network to describe the relation of all lanes, override this func to assign one when
        you are building your own block.
        return: roadnetwork
        """
        raise NotImplementedError

    def destroy(self):
        if self.block_network is not None:
            self.block_network.destroy()
            if self.block_network.graph is not None:
                self.block_network.graph.clear()
            self.block_network = None
        self.PART_IDX = 0
        self.ROAD_IDX = 0
        self._respawn_roads.clear()
        self._global_network = None
        super(BaseBlock, self).destroy()

    def __del__(self):
        self.destroy()
        logger.debug("{} is being deleted.".format(type(self)))

    @property
    def bounding_box(self):
        return self._bounding_box


"""
Digita endpoint.

You must declare environment variable DIGITA_URL to activate this plugin.

"""

import os
import json
import logging
import binascii
import dateutil
import pytz
from django.conf import settings
from django.conf.urls import url
from django.http import HttpResponse
from django.views.decorators.csrf import csrf_exempt
from django.utils import timezone
from influxdb.exceptions import InfluxDBClientError
from endpoints.utils import BasePlugin
from endpoints.utils import get_influxdb_client, create_influxdb_obj
from endpoints.utils import get_setting, get_datalogger
from endpoints.views import dump_request

ENV_NAME = 'DIGITA_URL'
URL = get_setting(ENV_NAME)
DIGITA_DB = get_setting('DIGITA_DB', 'digita')
logger = logging.getLogger(__name__)

def hex2int(hex_str):
    """
    Convert 2 hex characters (e.g. "23") to int (35)
    :param hex_str: hex character string
    :return: int integer
    """
    return int(hex_str, 16)

def calc_temp(hex_str):
    """
    Convert 4 hex characters (e.g. "040b") to float temp (25.824175824175825)
    :param hex_str: hex character string
    :return: float temperature
    """
    adc = int(hex_str[0:2], 16) * 256 + int(hex_str[2:4], 16)
    temp = (300 * adc / 4095) - 50
    return temp

def calc_volts(hex_str):
    """
    Convert 2 hex characters (e.g. "fe") to float volts (3.5043478260869567)
    :param hex_str: hex character string
    :return: float volts
    """
    volts = ((int(hex_str, 16) / 0.23) + 2400) / 1000
    return volts

def handle_clickey_tempsens(hex_str):
    temp1 = calc_temp(hex_str[2:6])
    temp2 = calc_temp(hex_str[6:10])
    volts = calc_volts(hex_str[10:12])
    return {
        'temp1': temp1,
        'temp2': temp2,
        'volt': volts
    }

def hex2value10(hex_str):
    return hex2int(hex_str) / 10.0

def handle_aqburk(hex_str):
    """
    Parse payload like "2a2a0021002c002800300056003b0000" float values
    :param hex_str: AQLoRaBurk hex payload
    :return: dict containing float values
    """
    data = {
        'pm25min': hex2value10(hex_str[4:8]),
        'pm25max': hex2value10(hex_str[8:12]),
        'pm25avg': hex2value10(hex_str[12:16]),
        'pm25med': hex2value10(hex_str[16:20]),
        'pm10min': hex2value10(hex_str[20:24]),
        'pm10max': hex2value10(hex_str[24:28]),
        'pm10avg': hex2value10(hex_str[28:32]),
        'pm10med': hex2value10(hex_str[32:36]),
    }
    if len(hex_str) == 52:
        data['temp'] = round(hex2value10(hex_str[36:40]) - 100, 1)
        data['humi'] = hex2value10(hex_str[40:44])
        data['pres'] = hex2value10(hex_str[44:48])
        data['gas'] = hex2value10(hex_str[48:52])
    return data

def handle_keyval(hex_str):
    """
    :param hex_str: key-value hex payload
    :return: dict containing parsed balues
    :raises UnicodeDecodeError: if hex_str contains illegal bytes for utf8
    """
    _str = binascii.unhexlify(hex_str)  # --> b'temp=24.61,hum=28.69'
    _str = _str.decode()  # --> 'temp=24.61,hum=28.69'
    keyvals = [x.split('=') for x in _str.split(',')]  # --> [['temp', '24.61'], ['hum', '28.69']]
    keyvals = [[x[0], float(x[1])] for x in keyvals]  # --> [['temp', 24.61], ['hum', 28.69]]
    data = dict(keyvals)  # --> {'temp': 24.61, 'hum': 28.69}
    return data

class Plugin(BasePlugin):
    """
    Digita plugin. Checks if endpoint's URL has been set in env.
    """
    name = 'digita'
    viewname = 'digitahandler'

    def __init__(self):
        """Check that `ENV_NAME` is in env variables."""
        super().__init__()
        if URL is not None:
            self.in_use = True

    def register(self):
        print('Registering plugin "{}"'.format(self.name))

    def get_urlpatterns(self):
        if self.in_use is False:
            print('{} environment variable is not set. {} endpoint is not in use.'.format(ENV_NAME, self.name))
            urlpatterns = []
        else:
            url_pattern = r'^{}$'.format(URL)
            urlpatterns = [
                url(url_pattern, self.view_func, name=self.viewname),
            ]
        return urlpatterns

    @csrf_exempt
    def view_func(self, request):
        """
        Endpoint requires valid Digita formatted JSON payload.
        """
        err_msg = ''
        status = 200
        try:
            body_data = request.body
            data = json.loads(body_data.decode('utf-8'))
        except (json.decoder.JSONDecodeError, UnicodeDecodeError) as err:
            log_msg = '[DIGITA] Invalid data: "{}". Hint: should be UTF-8 json.'.format(body_data[:50])
            err_msg = 'Invalid data: "{}"... Hint: should be UTF-8 json.'.format(body_data[:50])
            logger.error(log_msg)
            return HttpResponse(err_msg, status=400)
        # meta and type keys should be always in request json
        try:
            d = data['DevEUI_uplink']
            device = d['DevEUI']
            times = str(d['Time'])
            rssi = d['LrrRSSI']
            payload_hex = d['payload_hex']
        except KeyError as err:
            log_msg = '[DIGITA] Invalid json structure: "{}". Missing key: {}.'.format(body_data, err)
            err_msg = 'Invalid json structure: "{}". Hint: missing key {}.'.format(body_data, err)
            logger.error(log_msg)
            return HttpResponse(err_msg, status=400)
        now = timezone.now().astimezone(pytz.utc)
        path = os.path.join(settings.MEDIA_ROOT, 'digita', now.strftime('%Y-%m-%d'), device)
        os.makedirs(path, exist_ok=True)
        fpath = os.path.join(path, now.strftime('%Y%m%dT%H%M%S.%fZ.json'))
        with open(fpath, 'wt') as destination:
            destination.write(json.dumps(data, indent=1))
        response = HttpResponse("ok")
        # TODO: move this to a function
        if len(payload_hex) == 8:
            idata = {
                'wifi': int(payload_hex[0:4], 16),
                'ble': int(payload_hex[4:8], 16)
            }
            idata['rssi'] = rssi
            keys_str = 'wifi-ble'
            dl_descr = 'paxcounter'
            datalogger, created = get_datalogger(device, description=dl_descr, update_activity=True)
            ts = dateutil.parser.parse(times)
            measurement = create_influxdb_obj(device, keys_str, idata, ts)
            measurements = [measurement]
            # dbname = request.GET.get('db', DIGITA_DB)
            dbname = 'paxcounter'
            iclient = get_influxdb_client(database=dbname)
            iclient.create_database(dbname)
            try:
                iclient.write_points(measurements)
            except InfluxDBClientError as err:
                err_msg = '[DIGITA] InfluxDB error: {}'.format(err)
                status = 500
        elif payload_hex[:2] == '13':
            idata = handle_clickey_tempsens(payload_hex)
            idata['rssi'] = rssi
            keys_str = 'tempsens'
            datalogger, created = get_datalogger(device, description='Clickey Tempsens PRO', update_activity=True)
            ts = dateutil.parser.parse(times)
            measurement = create_influxdb_obj(device, keys_str, idata, ts)
            measurements = [measurement]
            # dbname = request.GET.get('db', DIGITA_DB)
            dbname = 'digita'
            iclient = get_influxdb_client(database=dbname)
            iclient.create_database(dbname)
            try:
                iclient.write_points(measurements)
            except InfluxDBClientError as err:
                err_msg = '[DIGITA] InfluxDB error: {}'.format(err)
                status = 500
        elif payload_hex[:2].lower() == '2a':  # payload_hex[:4].lower() == '2a2a':
            idata = handle_aqburk(payload_hex)
            idata['rssi'] = rssi
            keys_str = 'aqburk'
            datalogger, created = get_datalogger(device, description='FVH AQ burk', update_activity=True)
            ts = dateutil.parser.parse(times)
            measurement = create_influxdb_obj(device, keys_str, idata, ts)
            measurements = [measurement]
            DIGITA_DB = 'aqburk'
            dbname = request.GET.get('db', DIGITA_DB)
            iclient = get_influxdb_client(database=dbname)
            iclient.create_database(dbname)
            try:
                iclient.write_points(measurements)
            except InfluxDBClientError as err:
                err_msg = '[DIGITA] InfluxDB error: {}'.format(err)
                status = 500
        elif len(payload_hex) >= 2:  # Assume we have key-val data
            try:
                idata = handle_keyval(payload_hex)
            except (UnicodeDecodeError, IndexError) as err:
                err_msg = '[DIGITA] Payload error: {}'.format(err)
                status = 400
                logger.error(err_msg)
                dump_request(request, postfix='digita')
                response = HttpResponse(err_msg, status=status)
                return response
            idata['rssi'] = rssi
            ikeys = list(idata.keys())
            ikeys.sort()
            keys_str = '_'.join(ikeys)
            datalogger, created = get_datalogger(device, description='LoRaWAN device', update_activity=True)
            ts = dateutil.parser.parse(times)
            measurement = create_influxdb_obj(device, keys_str, idata, ts)
            measurements = [measurement]
            # dbname = request.GET.get('db', DIGITA_DB)
            dbname = 'digita'
            iclient = get_influxdb_client(database=dbname)
            iclient.create_database(dbname)
            try:
                iclient.write_points(measurements)
            except InfluxDBClientError as err:
                err_msg = '[DIGITA] InfluxDB error: {}'.format(err)
                status = 500
        else:
            err_msg = '[DIGITA] Not handled'
        if err_msg != '':
            logger.error(err_msg)
            dump_request(request, postfix='digita')
            response = HttpResponse(err_msg, status=status)
        return response