# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from marionette_driver.errors import JavascriptException

from marionette_harness import MarionetteTestCase

class TestExecuteSandboxes(MarionetteTestCase):
    def setUp(self):
        super(TestExecuteSandboxes, self).setUp()

    def test_execute_system_sandbox(self):
        # Test that "system" sandbox has elevated privileges in execute_script
        result = self.marionette.execute_script(
            "return Components.interfaces.nsIPermissionManager.ALLOW_ACTION",
            sandbox="system",
        )
        self.assertEqual(result, 1)

    def test_execute_async_system_sandbox(self):
        # Test that "system" sandbox has elevated privileges in
        # execute_async_script.
        result = self.marionette.execute_async_script(
            """
            let result = Ci.nsIPermissionManager.ALLOW_ACTION;
            arguments[0](result);
        """,
            sandbox="system",
        )
        self.assertEqual(result, 1)

    def test_execute_switch_sandboxes(self):
        # Test that sandboxes are retained when switching between them
        # for execute_script.
        self.marionette.execute_script("foo = 1", sandbox="1")
        self.marionette.execute_script("foo = 2", sandbox="2")
        foo = self.marionette.execute_script(
            "return foo", sandbox="1", new_sandbox=False
        )
        self.assertEqual(foo, 1)
        foo = self.marionette.execute_script(
            "return foo", sandbox="2", new_sandbox=False
        )
        self.assertEqual(foo, 2)

    def test_execute_new_sandbox(self):
        # test that clearing a sandbox does not affect other sandboxes
        self.marionette.execute_script("foo = 1", sandbox="1")
        self.marionette.execute_script("foo = 2", sandbox="2")

        # deprecate sandbox 1 by asking explicitly for a fresh one
        with self.assertRaises(JavascriptException):
            self.marionette.execute_script(
                """
                return foo
            """,
                sandbox="1",
                new_sandbox=True,
            )

        foo = self.marionette.execute_script(
            "return foo", sandbox="2", new_sandbox=False
        )
        self.assertEqual(foo, 2)

    def test_execute_async_switch_sandboxes(self):
        # Test that sandboxes are retained when switching between them
        # for execute_async_script.
        self.marionette.execute_async_script("foo = 1; arguments[0]();", sandbox="1")
        self.marionette.execute_async_script("foo = 2; arguments[0]();", sandbox="2")
        foo = self.marionette.execute_async_script(
            "arguments[0](foo);", sandbox="1", new_sandbox=False
        )
        self.assertEqual(foo, 1)
        foo = self.marionette.execute_async_script(
            "arguments[0](foo);", sandbox="2", new_sandbox=False
        )
        self.assertEqual(foo, 2)

class TestExecuteSandboxesChrome(TestExecuteSandboxes):
    def setUp(self):
        super(TestExecuteSandboxesChrome, self).setUp()
        self.marionette.set_context("chrome")


#!/usr/bin/env python3
"""
http://62.75.216.162:24443/amzn1.ask.account.AHYHW2CISNL5BA64VV7VNSJWYB3PNPWV5DRZCLUQLX46KYGAVJNPORBXUDOAVRLSVEEYYXFK3XYIEJHA6SM6WKBHIKTY6E4WD3NXB4VVJHDIEBGDDU5YBBKHDQRXLNB3H55LIMNZKBZD5MVYTY3UC3V4L7IJETPPW6UA7B5YBDF3GFJCY7MPYCCQC5LSAI4WSQEEZVXBZLZILBA/room/
"""

import json
import mimetypes
import re
import socketserver
import signal
import ssl
import sys
import threading

from datetime import time, datetime
from _socket import socket, AF_INET, SOCK_DGRAM
from time import sleep

from http.server import BaseHTTPRequestHandler, HTTPServer

class RequestsStore:
    def __init__(self, id, request):
        self.id = id
        self.request = request
        self.expecting_data = False
        self.next_response = False
        self.running_id = 0
        self.next_keep_alive = 10
        self.next_request = None

clients = dict()

def get_base_port():
    return 24443

def get_template(filename):
    with open(filename, "rb") as f:
        r = f.read()
    return r.decode("utf-8")

rewrite_pages = [  # const
    ['^/(.*)(html|ico|js|ttf|svg|woff|eot|otf|css|less|map).*$', './\\1\\2'],
    ['^/$', './index.html']
]

def translate_id(user_id):
    print("translate_id {}".format(user_id))
    s = open("alexa-forwardermaps.json", 'r').read()
    map_of_ids = json.loads(s)
    try:
        return map_of_ids[user_id]
    except:
        return None

class RequestHandler(BaseHTTPRequestHandler):
    def log_message(self, format, *args):
        return

    def page_not_found(self):
        self.send_response(404)
        self.send_header("Content-type", "application/json")
        self.end_headers()
        output = '{"errors":[{"status": 404, "message": "page not found: ' + self.path + '"}]}'
        self.wfile.write(bytearray(output, 'UTF-8'))

    def send_json_response(self, json_string):
        try:
            self.send_response(200)
            self.send_header("Content-type", "application/json")
            self.end_headers()
            self.wfile.write(bytearray(json_string, 'UTF-8'))
        except Exception as e:
            print("handle_get_query error {0}".format(e))

    def do_POST(self):
        print("Post request: {}".format(self.path))
        ctype, pdict = cgi.parse_header(self.headers.getheader('content-type'))
        print(ctype)
        print(pdict)

    def do_GET(self):
        print("Get request: {}".format(self.path))
        try:
            for pair in rewrite_pages:
                key = pair[0]
                replacement = pair[1]
                if re.match(key, self.path):
                    path = re.sub(key, replacement, self.path)
                    try:
                        print(path)
                        output = open(path, 'rb').read()
                        self.send_response(200)
                        guessed_mime_type = mimetypes.guess_type(path, strict=False)
                        print(path, guessed_mime_type)
                        self.send_header("Content-type", guessed_mime_type)
                        self.end_headers()
                        self.wfile.write(output)
                        return
                    except Exception as e:
                        print("Exception {0}".format(e))
                        pass
        except:
            pass
        try:
            if self.path == './html/index.html':
                try:
                    self.send_response(200)
                    self.send_header("Content-type", "text/html")
                    self.end_headers()
                    self.wfile.write(get_template('index.html'), 'UTF-8')
                    return
                except Exception as e:
                    self.send_response(200)
                    self.send_header("Content-type", "text/html")
                    self.end_headers()
                    output = "Rephrase your request, this is 404
you asked for [" + str(self.path) + "]"
                    self.wfile.write(bytearray(output, 'UTF-8'))
                    return
            else:
                result = send_infos(self.path)
                if result == "not allowed":
                    output = b'{"result":"not allowed","reason":"no active client connected to the forwarder with your id. Or you are a teapot."}'
                    self.send_response(401)
                    self.send_header("Content-type", "application/json")
                    self.end_headers()
                else:
                    output = result.encode('utf-8')
                    self.send_response(200)
                    self.send_header("Content-type", "application/json")
                    self.end_headers()
                self.wfile.write(output)
                return

            output = open("./html/" + self.path[1:], 'r').read()
            self.send_response(200)
            self.send_header("Content-type", "application/json")
            self.end_headers()
            self.wfile.write(b"\n")
            self.wfile.write(bytearray(output, 'UTF-8'))
        except Exception as e:
            print("Error ", e)
            self.send_response(404)
            self.send_header("Content-type", "text/html")
            self.end_headers()
            output = "Rephrase your request, this is 404
you asked for [" + str(self.path) + "]"
            self.wfile.write(bytearray(output, 'UTF-8'))

def send_infos(path):
    global clients
    try:
        r = path.split('/', 2)
        user_id = translate_id(r[1])
        print("Voice message from user {}".format(user_id))
        clients[user_id].next_keep_alive = 10
        clients[user_id].expecting_data = True
        clients[user_id].running_id += 1
        msg = "#alexa {} /text{}\n".format(clients[user_id].running_id, path[len(r[1])+1:])
        print("sending msg: " + msg)
        clients[user_id].next_response = None
        clients[user_id].next_request = bytearray(msg.encode('Utf-8'))
        sleep(1)
        while clients[user_id].next_response is None:
            print("waiting for response")
            sleep(1)
        response_data = clients[user_id].next_response

#        return "ok"
#        response_data = clients[user_id].request.recv(1500).decode('utf-8')
        print("####got response from pfserver: '{}'".format(response_data))
        if response_data.startswith("#ack {} ".format(clients[user_id].running_id)):
            clients[user_id].expecting_data = False
            return response_data.split(" ", 2)[2]
        else:
            clients[user_id].expecting_data = False
            return response_data

    except Exception as e:
        print("send_infos error occured: {}".format(e))
        clients[user_id].expecting_data = False
        return "not allowed"

def scan_raumfeld():
    global clients
    while 1:
        for key, value in clients.items():
            if value.next_request is not None:
                value.next_response = None
                value.request.sendall(value.next_request)
                value.next_request = None
            if value.next_keep_alive == 0:
                value.next_keep_alive = 10
                msg = '#keep-alive ' + key + " " + datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f") + '\n'
                print(msg)
                try:
                    value.request.sendall(bytearray(msg.encode('Utf-8')))
                except:
                    pass
            value.next_keep_alive -= 1

        sleep(1)

def signal_handler(signal, frame):
    global fetcher_server
    print('received signal', signal)
    print("shutting down connections")
    for key, value in clients.items():
        value.request.close()
    fetcher_server.shutdown()
    http_server.shutdown()
    sys.exit(0)

signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
print('Press Ctrl+C')

threading.Thread(target=scan_raumfeld).start()

class ThreadedTCPRequestHandler(socketserver.BaseRequestHandler):
    def handle(self):
        global clients
        print("got a connection from ", self.client_address)
        self.request.settimeout(5)
        user_id = None
        end_connection = False
        error_count = 0
        empty_count = 0
        self.request.sendall(b'login:\r\n')
        while True:
            try:
                raw_data = self.request.recv(1500)
            except:
                raw_data = b""
            if len(raw_data) == 0:
                empty_count += 1
                if empty_count > 120/5:
                    print("receiving rubbish, terminating connection")
                    self.request.sendall(b'bye bye :(\r\n')
                    self.request.close()
                    return
            else:
                empty_count = 0
                data = str(raw_data, 'utf-8').strip()

                print("I received '{}'".format(data))
                if data.startswith('#id '):
                    if user_id is not None:
                        self.request.sendall(b'You already gave an ID!\r\n')
                    else:
                        user_id = data.split(' ')[1]
                        self.request.sendall(b'password:\r\n')
                if data.startswith('#pwd '):
                    if user_id is not None:
                        print("client identified with id {}. Adding to pool.".format(user_id))
                        clients[user_id] = RequestsStore(user_id, self.request)
                        self.request.sendall(b'ok\r\n')

                    else:
                        print("sends password (which is wrong).".format(user_id))
                        end_connection = True

                if user_id is None:
                    error_count += 1
                    if error_count == 3:
                        end_connection = True
                    self.request.sendall(b'asked you to login:\r\n')

                if data.startswith('#quit'):
                    self.request.sendall(b'see you! :)\r\n')
                    self.request.close()
                    try:
                        del clients[user_id]
                    except KeyError:
                        pass
                    print("Client exit!")
                    return

                if data.startswith('#ack '):
                    clients[user_id].next_response = data
                    print("response is {}".format(data))
#                    self.request.send(b'ok\r\n')
#                    self.request.send(data.encode('utf-8'))

                if end_connection:
                    self.request.sendall(b'bye bye :(\r\n')
                    self.request.close()
                    return

class ThreadedTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
    pass

class ThreadedHTTPServer(socketserver.ThreadingMixIn, HTTPServer):
    pass

def get_local_ip_address():
    s = socket(AF_INET, SOCK_DGRAM)
    s.connect(("8.8.8.8", 80))
    return s.getsockname()[0]

if __name__ == "__main__":
    socketserver.TCPServer.allow_reuse_address = True

    host, server_port = get_local_ip_address(), get_base_port()
    fetcher_server = ThreadedTCPServer((host, server_port+2), ThreadedTCPRequestHandler)
    http_server = ThreadedHTTPServer((host, server_port), RequestHandler)
    print("Started fetcher server as {0}:{1}".format(host, server_port+2))
    print("Started http server as {0}:{1}".format(host, server_port))

    fetcher_server_thread = threading.Thread(target=fetcher_server.serve_forever)
#    fetcher_server_thread.daemon = True
    fetcher_server_thread.start()

    server_thread = threading.Thread(target=http_server.serve_forever)
 #   server_thread.daemon = True
    server_thread.start()

    fetcher_server_thread.join()
    server_thread.join()

    fetcher_server.shutdown()
    fetcher_server.server_close()
    http_server.shutdown()
    http_server.server_close()
    print("stopped all")


"""
overwrite_checkpoint.py: overwrite existing key and value in ZephIR checkpoint.

Usage:
	overwrite_checkpoint.py -h | --help
	overwrite_checkpoint.py -v | --version
	overwrite_checkpoint.py --dataset=<dataset> --key=<key> --value=<value> [options]

Options:
	-h --help                           	show this message and exit.
	-v --version                        	show version information and exit.
	--dataset=<dataset>  					path to data directory to analyze.
	--key=<key>  							name of item in checkpoint to overwrite.
	--value=<value>  						new item to write into checkpoint.
"""

from docopt import docopt

from ..__version__ import __version__

from . import *
from ..models.container import Container
from ..utils.utils import *

def overwrite_checkpoint(dataset, key, value):
	_checkpoint = load_checkpoint(dataset, fallback=False, verbose=True)

	updated_items = 0

	if key == 'args' and value in ['all', '', None]:
		if (dataset / 'args.json').is_file():
			with open(str(dataset / 'args.json')) as json_file:
				args = json.load(json_file)
		update_checkpoint(dataset, {'args': args})
		updated_items += 1

	if key in _checkpoint:
		print('Updating checkpoint main dict...')
		update_checkpoint(dataset, {key: type(_checkpoint[key])(value)})
		updated_items += 1

	if key in _checkpoint['args'] or f'--{key}' in _checkpoint['args']:
		print('Updating saved args...')
		args = _checkpoint['args']
		args[f'--{key}'] = type(args[f'--{key}'])(value)
		update_checkpoint(dataset, {'args': args})
		updated_items += 1

		_args_list = [
			'allow_rotation', 'channel', 'exclude_self', 'exclusive_prov',
			'gamma', 'include_all', 'n_frame', 'z_compensator'
		]
		if key in _args_list:
			print('Rebuilding container from updated args...')
			container = Container(
				dataset=dataset,
				allow_rotation=args['--allow_rotation'] in ['True', 'Y', 'y'],
				channel=int(args['--channel']) if args['--channel'] else None,
				dev='cpu',
				exclude_self=args['--exclude_self'] in ['True', 'Y', 'y'],
				exclusive_prov=(bytes(args['--exclusive_prov'], 'utf-8')
								if args['--exclusive_prov'] else None),
				gamma=float(args['--gamma']),
				include_all=args['--include_all'] in ['True', 'Y', 'y'],
				n_frame=int(args['--n_frame']),
				z_compensator=float(args['--z_compensator']),
			)
			updated_items += 1
		else:
			container = _checkpoint['container']
		_args_list = _args_list + ['t_ref', 'wlid_ref', 'n_ref']
		if key in _args_list:
			build_annotations(
				container=container,
				annotation=None,
				t_ref=eval(args['--t_ref']) if args['--t_ref'] else None,
				wlid_ref=eval(args['--wlid_ref']) if args['--wlid_ref'] else None,
				n_ref=int(args['--n_ref']) if args['--n_ref'] else None,
			)
			updated_items += 1
		_args_list = _args_list + ['dimmer_ratio', 'grid_shape', 'fovea_sigma', 'n_chunks']
		if key in _args_list:
			build_models(
				container=container,
				dimmer_ratio=float(args['--dimmer_ratio']),
				grid_shape=(5, 2 * (int(args['--grid_shape']) // 2) + 1,
							2 * (int(args['--grid_shape']) // 2) + 1),
				fovea_sigma=(1, float(args['--fovea_sigma']),
							 float(args['--fovea_sigma'])),
				n_chunks=int(args['--n_chunks']),
			)
			updated_items += 1
		_args_list = _args_list + ['load_nn', 'nn_max']
		if key in _args_list:
			build_springs(
				container=container,
				load_nn=args['--load_nn'] in ['True', 'Y', 'y'],
				nn_max=int(args['--nn_max']),
			)
			updated_items += 1
		_args_list = _args_list + ['sort_mode', 't_ignore']
		if key in _args_list:
			build_tree(
				container=container,
				sort_mode=str(args['--sort_mode']),
				t_ignore=eval(args['--t_ignore']) if args['--t_ignore'] else None,
			)
			updated_items += 1

	if updated_items == 0:
		print('*** KEY NOT FOUND! Check arguments and try again.')

	return

def main():
	args = docopt(__doc__, version=f'ZephIR overwrite_checkpoint {__version__}')
	print(args, '\n')

	overwrite_checkpoint(
		dataset=Path(args['--dataset']),
		key=str(args['--key']),
		value=eval(args['--value']),
	)

if __name__ == '__main__':
	main()


from PIL import ImageDraw, Image, ImageFont
from random import randint
from io import BytesIO
from db import get_player_by_id

from discord.ext.commands.converter import TextChannelConverter

def centered_table(w, h, txt, color):
    txtbox = Image.new('RGB', (w, h), color)
    drawtxt = ImageDraw.Draw(txtbox)
    drawtxt.rectangle((0,0,w-1,h-1), outline=1)
    font = ImageFont.truetype('./arial.ttf', 18)
    wt, ht = drawtxt.textsize(txt, font=font)
    drawtxt.text(((w-wt)//2,(h-ht)//2), txt, font=font, fill="black")
    return txtbox

def load_map(players_data, env_data):
    sz = env_data['size'] + 1
    map = [[-1 for i in range(sz - 1)] for i in range(sz - 1)]
    for x in players_data:
        if x['pos'] != '':
            pos = x['pos'].split(':')
            map[int(pos[1]) - 1][int(pos[0]) - 1] = x['id']
    rectSize = 40
    w = sz * rectSize + 1
    h = sz * rectSize + 1
    img = Image.new('RGB', (w, h), 'white')
    draw = ImageDraw.Draw(img)
    fpx, fpy = 0, rectSize
    font = ImageFont.truetype('./arial.ttf', 20)
    lit = 1
    draw.rectangle([0, 0, rectSize, rectSize], outline=1)
    for i in range(sz):
        draw.rectangle([fpx, fpy, fpx+rectSize, fpy+rectSize], outline=1)
        draw.text((fpx + rectSize // 4, fpy + rectSize // 4), str(lit), font=font, fill='black')
        fpy += rectSize
        lit += 1
    lit = 'A'
    fpx, fpy = rectSize, 0
    for i in range(sz):
        draw.rectangle([fpx, fpy, fpx+rectSize, fpy+rectSize], outline=1)
        draw.text((fpx + rectSize // 4, fpy + rectSize // 4), lit, font=font, fill='black')
        fpx += rectSize
        lit = chr(ord(lit) + 1)
    fpx, fpy = rectSize, rectSize
    colors = {}
    font = ImageFont.truetype('./arial.ttf', 16)
    for i in range(sz - 1):
        for j in range(sz - 1):
            if (map[i][j] == -1):
                draw.rectangle([fpx, fpy, fpx+rectSize, fpy+rectSize], outline=1)
            else:
                r, g, b = (int(str(map[i][j])[::-1]) * 1244 + 144) * 3 % 255, ((map[i][j] * 5352 + 42311) * 14221) % 255, ((map[i][j] * 1244 + 144) * 3) % 255
                draw.rectangle([fpx, fpy, fpx+rectSize, fpy+rectSize], fill=(r, g, b), outline=1)
                pl = get_player_by_id(str(map[i][j]))
                draw.text((fpx + 2, fpy + 6), pl['name'][0:3], font=font, stroke_width=1, stroke_fill='black')
                colors[map[i][j]] = (r, g, b)
            fpx += rectSize
        fpx = rectSize
        fpy += rectSize
    widthes = [30*14, 30*4-30, 30*2-4, 30*5+6, 30*5-2]
    table = Image.new('RGB', (1000, 32 * (len(players_data) + 1) - 2), 'white')
    fpx, fpy = 0, 0
    table.paste(centered_table(widthes[0], 32, 'Игрок', (255, 255, 255)), (fpx, fpy))
    fpx += 30 * 14 - 1
    table.paste(centered_table(widthes[1], 32, 'Здоровье', (255, 255, 255)), (fpx, fpy))
    fpx += 30*4-30 - 1
    table.paste(centered_table(widthes[2], 32, 'ОД', (255, 255, 255)), (fpx, fpy))
    fpx += 30 * 2 - 4 - 1
    table.paste(centered_table(widthes[3], 32, 'Получил помощи', (255, 255, 255)), (fpx, fpy))
    fpx += 30 * 5 + 6 - 1
    table.paste(centered_table(widthes[4], 32, 'Оказал помощи', (255, 255, 255)), (fpx, fpy))
    fpx += 30 * 5 - 2 - 1
    table.paste(centered_table(1000 - fpx, 32, 'Нанес урона', (255, 255, 255)), (fpx, fpy))
    widthes.append(1000 - fpx)
    fpy += 32
    fpx = 0
    for p in players_data:
        i = 0
        for x in p:
            if x not in ["id", "pos"] and p['pos'] != '':
                table.paste(centered_table(widthes[i], 32, str(p[x]), colors[p['id']]), (fpx, fpy))
                fpx += widthes[i] - 1
                i += 1
        fpx, i = 0, 0
        fpy += 32 - 1
    ret = Image.new('RGB', (img.width + table.width, max(img.height, table.height)), 'white')
    ret.paste(img, (0, 0))
    ret.paste(table, (img.width, 0))

    buffer = BytesIO()
    ret.save(buffer,format="PNG")
    buffer.seek(0)

    return buffer


from django.shortcuts import render, get_object_or_404

from . models import Category, Product
from cart.forms import CartAddProductForm

# list all products or filter products by a given category
def product_list(request, category_slug=None):
	category = None
	categories = Category.objects.all()
	products = Product.objects.filter(available=True)
	if category_slug:
		category = get_object_or_404(Category, slug=category_slug)
		products = products.filter(category=category)
	context = {
		'category': category,
		'categories': categories,
		'products': products
	}
	return render(request, 'shop/product/list.html', context)

def product_detail(request, id, slug):
	# *arg being id. slug and available being **kwargs)
	# we can get the instance just with id. slug is included to build SEO-friendly URLs for products
	# try-except would be the alternative approach to get_object_or_404
	product = get_object_or_404(Product, id=id, slug=slug, available=True)

	cart_product_form = CartAddProductForm()

	# retrieving product with id only
	#product = products.objects.filter(id)

	context = {
		'product': product,
		'cart_product_form': cart_product_form
	}
	return render(request, 'shop/product/detail.html', context)


# !/usr/bin/env python
# coding: utf-8

"""
Some utility functions aiming to analyse OSM data
"""

import datetime as dt
from datetime import timedelta
import re
import math

import numpy as np
import pandas as pd
import statsmodels.api as sm

from osmdq.extract_user_editor import editor_name

### OSM data exploration ######################
def updatedelem(data):
    """Return an updated version of OSM elements

    Parameters
    ----------
    data: df
        OSM element timeline

    """
    updata = data.groupby(['elem','id'])['version'].max().reset_index()
    return pd.merge(updata, data, on=['id','version'])

def datedelems(history, date):
    """Return an updated version of history data at date

    Parameters
    ----------
    history: df
        OSM history dataframe
    date: datetime
        date in datetime format

    """
    datedelems = (history.query("ts <= @date")
                  .groupby(['elem','id'])['version']
                  .max()
                  .reset_index())
    return pd.merge(datedelems, history, on=['elem','id','version'])

def osm_stats(osm_history, timestamp):
    """Compute some simple statistics about OSM elements (number of nodes,
    ways, relations, number of active contributors, number of change sets

    Parameters
    ----------
    osm_history: df
        OSM element up-to-date at timestamp
    timestamp: datetime
        date at which OSM elements are evaluated
    """
    osmdata = datedelems(osm_history, timestamp)
    nb_nodes = len(osmdata.query('elem=="node"'))
    nb_ways = len(osmdata.query('elem=="way"'))
    nb_relations = len(osmdata.query('elem=="relation"'))
    nb_users = osmdata.uid.nunique()
    nb_chgsets = osmdata.chgset.nunique()
    return [nb_nodes, nb_ways, nb_relations, nb_users, nb_chgsets]

def osm_chronology(history, start_date, end_date=dt.datetime.now()):
    """Evaluate the chronological evolution of OSM element numbers

    Parameters
    ----------
    history: df
        OSM element timeline

    """
    timerange = pd.date_range(start_date, end_date, freq="1M").values
    osmstats = [osm_stats(history, str(date)) for date in timerange]
    osmstats = pd.DataFrame(osmstats, index=timerange,
                            columns=['n_nodes', 'n_ways', 'n_relations',
                                     'n_users', 'n_chgsets'])
    return osmstats

### OSM metadata extraction ####################
def group_count(metadata, data, grp_feat, res_feat, namesuffix):
    """Group-by 'data' by 'grp_feat' and element type features, count element
    corresponding to each grp_feat-elemtype tuples and merge them into metadata
    table

    Parameters
    ----------
    metadata: df
        Dataframe that will integrate the new features
    data: df
        Dataframe from where information is grouped
    grp_feat: object
        string that indicates which feature from 'data' must be used to group items
    res_feat: object
        string that indicates the measured feature (how many items correspond
    to the criterion)
    namesuffix: object
        string that ends the new feature name

    """
    md_ext = (data.groupby([grp_feat, 'elem'])[res_feat]
              .count()
              .unstack()
              .reset_index()
              .fillna(0))
    md_ext['elem'] = md_ext[['node','relation','way']].apply(sum, axis=1)
    md_ext = md_ext[[grp_feat, 'elem', 'node', 'way', 'relation']]
    colnames = "n_" + md_ext.columns.values[-4:] + namesuffix
    md_ext.columns = [grp_feat, *colnames]
    return pd.merge(metadata, md_ext, on=grp_feat, how='outer').fillna(0)

def group_nunique(metadata, data, grp_feat, res_feat, namesuffix):
    """Group-by 'data' by 'grp_feat' and element type features, count unique
    element corresponding to each grp_feat-elemtype tuples and merge them into
    metadata table

    Parameters
    ----------
    metadata: df
        Dataframe that will integrate the new features
    data: df
        Dataframe from where information is grouped
    grp_feat: object
        string that indicates which feature from 'data' must be used to group items
    res_feat: object
        string that indicates the measured feature (how many items correspond
    to the criterion)
    namesuffix: object
        string that ends the new feature name

    """
    md_ext = (data.groupby([grp_feat, 'elem'])[res_feat]
              .nunique()
              .unstack()
              .reset_index()
              .fillna(0))
    md_ext['elem'] = md_ext[['node','relation','way']].apply(sum, axis=1)
    md_ext = md_ext[[grp_feat, 'elem', 'node', 'way', 'relation']]
    colnames = "n_" + md_ext.columns.values[-4:] + namesuffix
    md_ext.columns = [grp_feat, *colnames]
    return pd.merge(metadata, md_ext, on=grp_feat, how='outer').fillna(0)

def group_stats(metadata, data, grp_feat, res_feat, nameprefix, namesuffix):
    """Group-by 'data' by 'grp_feat' and element type features, compute basic
    statistic features (first and ninth deciles) corresponding to each
    grp_feat-elemtype tuples and merge them into metadata table; need
    Pandas >= 0.20

    Parameters
    ----------
    metadata: df
        Dataframe that will integrate the new features
    data: df
        Dataframe from where information is grouped
    grp_feat: object
        string that indicates which feature from 'data' must be used to group items
    res_feat: object
        string that indicates the measured feature (how many items correspond
    to the criterion)
    nameprefix: object
        string that begins the new feature name
    namesuffix: object
        string that ends the new feature name

    """
    md_ext = (data.groupby(grp_feat)[res_feat]
              .quantile(q=[0.1, 0.9])
              .unstack()
              .reset_index())
    colnames = [nameprefix + str(int(100*op)) + namesuffix
                 for op in md_ext.columns.values[1:]]
    md_ext.columns = [grp_feat, *colnames]
    return pd.merge(metadata, md_ext, on=grp_feat, how='outer').fillna(0)

def init_metadata(osm_elements, init_feat, timeunit='1d'):
    """ This function produces an init metadata table based on 'init_feature'
    in table 'osm_elements'. The intialization consider timestamp measurements
    (generated for each metadata tables, i.e. elements, change sets and users).

    Parameters
    ----------
    osm_elements: pd.DataFrame
        OSM history data
    duration_feat: object
        metadata duration feature name in string format
    timeunit: object
        time unit in which 'duration_feature' will be expressed

    """
    metadata = (osm_elements.groupby(init_feat)['ts']
                .agg(["min", "max"])
                .reset_index())
    metadata.columns = [*init_feat, 'first_at', 'last_at']
    metadata['lifespan'] = ((metadata.last_at - metadata.first_at)
                            / pd.Timedelta(timeunit))
    extraction_date = osm_elements.ts.max()
    metadata['n_inscription_days'] = ((extraction_date - metadata.first_at)
                                      / pd.Timedelta('1d'))
    metadata['n_activity_days'] = (osm_elements
                                   .groupby(init_feat)['ts']
                                   .nunique()
                                   .reset_index())['ts']
    return metadata.sort_values(by=['first_at'])

def enrich_osm_elements(osm_elements):
    """Enrich OSM history data by computing additional features

    Parameters
    ----------
    osm_elements: pd.DataFrame
        OSM history data

    """
    # Extract information from first and last versions
    osmelem_first_version = (osm_elements
                             .groupby(['elem','id'])['version', 'uid']
                             .first()
                             .reset_index())
    osm_elements = pd.merge(osm_elements, osmelem_first_version,
                                on=['elem','id'])
    osm_elements.columns = ['elem', 'id', 'version', 'visible', 'ts',
                                'uid', 'chgset', 'vmin', 'first_uid']
    osmelem_last_version = (osm_elements
                             .groupby(['elem','id'])['version', 'uid',
                                                     'visible']
                             .last()
                             .reset_index())
    osm_elements = pd.merge(osm_elements, osmelem_last_version,
                                on=['elem','id'])
    osm_elements.columns = ['elem', 'id', 'version', 'visible', 'ts', 'uid',
                            'chgset', 'vmin', 'first_uid', 'vmax', 'last_uid',
                            'available']
    osmelem_last_bychgset = (osm_elements
                             .groupby(['elem','id','chgset'])['version',
                                                              'visible']
                             .last()
                             .reset_index())
    osm_elements = pd.merge(osm_elements,
                            osmelem_last_bychgset[['elem', 'id',
                                                   'chgset', 'visible']],
                            on=['elem','id', 'chgset'])
    osm_elements.columns = ['elem', 'id', 'version', 'visible', 'ts', 'uid',
                            'chgset', 'vmin', 'first_uid', 'vmax', 'last_uid',
                            'available', 'open']

    # New version-related features
    osm_elements['init'] = osm_elements.version == osm_elements.vmin
    osm_elements['up_to_date'] = osm_elements.version == osm_elements.vmax
    osm_elements = osm_elements.drop(['vmin'], axis=1)

    osmelem_first_bychgset = (osm_elements
                             .groupby(['elem','id','chgset'])['version', 'init']
                             .first()
                             .reset_index())
    osm_elements = pd.merge(osm_elements,
                            osmelem_first_bychgset[['elem', 'id',
                                                    'chgset', 'init']],
                            on=['elem','id','chgset'])
    osm_elements.columns = ['elem', 'id', 'version', 'visible', 'ts', 'uid',
                            'chgset', 'first_uid', 'vmax', 'last_uid',
                            'available', 'open', 'init', 'up_to_date',
                            'created']

    # Whether or not an element will be corrected in the last version
    osm_elements['willbe_corr'] = np.logical_and(osm_elements.id.diff(-1)==0,
                                              osm_elements.uid.diff(-1)!=0)
    osm_elements['willbe_autocorr'] = np.logical_and(osm_elements.id.diff(-1)==0,
                                                     osm_elements.uid
                                                     .diff(-1)==0)

    # Time before the next modification
    osm_elements['nextmodif_in'] = - osm_elements.ts.diff(-1)
    osm_elements.loc[osm_elements.up_to_date,['nextmodif_in']] = pd.NaT
    osm_elements.nextmodif_in = (osm_elements.nextmodif_in
                                 .astype('timedelta64[D]'))

    # Time before the next modification, if it is done by another user
    osm_elements['nextcorr_in'] = osm_elements.nextmodif_in
    osm_elements['nextcorr_in'] = (osm_elements.nextcorr_in
                                   .where(osm_elements.willbe_corr,
                                          other=pd.NaT))

    # Time before the next modification, if it is done by the same user
    osm_elements['nextauto_in'] = osm_elements.nextmodif_in
    osm_elements['nextauto_in'] = (osm_elements.nextauto_in
                                   .where(osm_elements.willbe_autocorr,
                                                   other=pd.NaT))

    return osm_elements

def extract_elem_metadata(osm_elements, user_groups, drop_ts=True):
    """ Extract element metadata from OSM history data

    Parameters
    ----------
    osm_elements: pd.DataFrame
        OSM history data

    Return
    ------
    elem_md: pd.DataFrame
        Change set metadata with timestamp information, version-related features
    and number of unique change sets (resp. users)

    """
    elem_md = init_metadata(osm_elements, ['elem','id'])
    elem_md['version'] = (osm_elements.groupby(['elem','id'])['version']
                          .max()
                          .reset_index())['version']
    elem_md['n_chgset'] = (osm_elements.groupby(['elem', 'id'])['chgset']
                           .nunique()
                           .reset_index())['chgset']
    elem_md['n_user'] = (osm_elements.groupby(['elem', 'id'])['uid']
                         .nunique()
                         .reset_index())['uid']
    elem_md['n_autocorr'] = (osm_elements
                             .groupby(['elem','id'])['willbe_autocorr']
                             .sum()
                             .reset_index()['willbe_autocorr']
                             .astype('int'))
    elem_md['n_corr'] = (osm_elements
                             .groupby(['elem','id'])['willbe_corr']
                             .sum()
                             .reset_index()['willbe_corr']
                             .astype('int'))
    elem_md = pd.merge(elem_md, osm_elements[['elem', 'id',
                                              'version', 'visible',
                                              'first_uid', 'last_uid']],
                       on=['elem', 'id', 'version'])
    elem_md = elem_md.set_index(['elem', 'id'])
    elem_md = elem_md.join(user_groups.Xclust, on='first_uid')
    elem_md = elem_md.rename(columns={'Xclust':'first_ug'})
    elem_md = elem_md.join(user_groups.Xclust, on='last_uid')
    elem_md = elem_md.rename(columns={'Xclust':'last_ug'})
    elem_md = elem_md.reset_index()
    if drop_ts:
        return drop_features(elem_md, '_at')
    else:
        return elem_md

def extract_chgset_metadata(osm_elements, drop_ts=True):
    """ Extract change set metadata from OSM history data

    Parameters
    ----------
    osm_elements: pd.DataFrame
        OSM history data

    Return
    ------
    chgset_md: pd.DataFrame
        Change set metadata with timestamp information, user-related features
    and other features describing modification and OSM elements themselves

    """
    chgset_md = init_metadata(osm_elements, ['chgset'], '1m')
    # User-related features
    chgset_md = pd.merge(chgset_md,
                         osm_elements[['chgset','uid']].drop_duplicates(),
                         on=['chgset'])
    chgset_md['user_lastchgset_h'] = (chgset_md.groupby('uid')['first_at']
                                      .diff())
    chgset_md.user_lastchgset_h = (chgset_md.user_lastchgset_h /
                                   timedelta(hours=1))
    chgset_md['user_chgset_rank'] = chgset_md.groupby('uid')['first_at'].rank()

    # Number of modifications per unique element
    contrib_byelem = (osm_elements.groupby(['elem', 'id', 'chgset'])['version']
                      .count()
                      .reset_index())
    chgset_md['nmean_modif_byelem'] = (contrib_byelem
                                       .groupby('chgset')['version']
                                       .mean()
                                       .reset_index())['version']
    # Element-related features
    chgset_md = extract_element_features(chgset_md, osm_elements,
                                       'node', 'chgset')
    chgset_md = extract_element_features(chgset_md, osm_elements,
                                       'way', 'chgset')
    chgset_md = extract_element_features(chgset_md, osm_elements,
                                       'relation', 'chgset')
    chset_md = chgset_md.set_index('chgset')
    if drop_ts:
        return drop_features(chgset_md, '_at')
    else:
        return chgset_md

def metadata_version(metadata, osmelem, grp_feat, res_feat, feature_suffix):
    """Compute the version-related features of metadata and append them into
    the metadata table

    Parameters
    ----------
    metadata: pd.DataFrame
        Metadata table to complete
    osmelem: pd.DataFrame
        original data used to compute versions; contains a 'elem' feature
    grp_feat: object
        string that indicates which feature from 'data' must be used to group items
    res_feat: object
        string that indicates the measured feature (how many items correspond
    feature_suffix: str
        string designing the end of the new feature names
    """
    osmelem_nodes = osmelem.query('elem=="node"')
    osmelem_ways = osmelem.query('elem=="way"')
    osmelem_relations = osmelem.query('elem=="relation"')
    metadata = group_stats(metadata, osmelem_nodes, grp_feat, res_feat,
                              'v', '_node'+feature_suffix)
    metadata = group_stats(metadata, osmelem_ways, grp_feat, res_feat,
                              'v', '_way'+feature_suffix)
    metadata = group_stats(metadata, osmelem_relations, grp_feat, res_feat,
                              'v', '_relation'+feature_suffix)
    return metadata

def extract_user_metadata(osm_elements, chgset_md, drop_ts=True):
    """ Extract user metadata from OSM history data

    Parameters
    ----------
    osm_elements: pd.DataFrame
        OSM history data
    chgset_md: pd.DataFrame
        OSM change set metadata

    Return
    ------
    user_md: pd.DataFrame
        User metadata with timestamp information, changeset-related features
    and other features describing modification and OSM elements themselves

    """
    user_md = init_metadata(osm_elements, ['uid'])
    # Change set-related features
    user_md['n_chgset'] = (chgset_md.groupby('uid')['chgset']
                           .count()
                           .reset_index())['chgset']
    user_md['dmean_chgset'] = (chgset_md.groupby('uid')['lifespan']
                               .mean()
                               .reset_index())['lifespan']
    # Number of modifications per unique element
    contrib_byelem = (osm_elements.groupby(['elem', 'id', 'uid'])['version']
                      .count()
                      .reset_index())
    user_md['nmean_modif_byelem'] = (contrib_byelem.groupby('uid')['version']
                                    .mean()
                                     .reset_index())['version']
    # Modification-related features
    user_md = extract_generic_modif_features(user_md, osm_elements, 'uid')
    user_md = extract_modif_features(user_md, osm_elements, 'node', 'uid')
    user_md = extract_modif_features(user_md, osm_elements, 'way', 'uid')
    user_md = extract_modif_features(user_md, osm_elements, 'relation', 'uid')
    user_md = user_md.set_index('uid')
    if drop_ts:
        return drop_features(user_md, '_at')
    else:
        return user_md

def add_chgset_metadata(metadata, total_change_sets):
    """Add total change set count to user metadata

    Parameters
    ----------
    metadata: pd.DataFrame
        user metadata; must be indexed by a column 'uid'
    total_change_sets: pd.DataFrame
        total number of change sets by user; must contain columns 'uid' and 'num'

    """
    metadata = (metadata.join(total_change_sets.set_index('uid'))
                .rename_axis({'num': 'n_total_chgset'}, axis=1))
    metadata['p_local_chgset'] = metadata.n_chgset / metadata.n_total_chgset
    return metadata

def add_editor_metadata(metadata, top_editors):
    """Add editor information to each metadata recordings; use an outer join to
    overcome the fact that some users do not indicate their editor, and may be
    skipped by a natural join => the outer join allow to keep them with 0
    values on known editors

    Parameters
    ----------
    metadata: pd.DataFrame
        user metadata; must be indexed by a column 'uid'
    top_editors: pd.DataFrame
        raw editor information, editors used by each user, with a highlight on
    N most popular editors; must contain a column 'uid'

    """
    metadata = (metadata
                .join(top_editors.set_index('uid'), how='left')
                .fillna(0))
    metadata['n_total_chgset_unknown'] = (metadata['n_total_chgset']
                                          - metadata['n_total_chgset_known'])
    return drop_features(metadata, 'n_total_chgset_known')

def transform_editor_features(metadata):
    """Transform editor-related features into metadata; editor uses are expressed
    as proportions of n_total_chgset, a proportion of local change sets is
    computed as a new feature and an ecdf transformation is applied on n_chgset
    and n_total_chgset

    Parameters
    ----------
    metadata: pd.DataFrame
        user metadata; must contain n_chgset and n_total_chgset columns, and
    editor column names must begin with 'n_total_chgset_'

    """
    normalize_features(metadata, 'n_total_chgset')
    metadata = ecdf_transform(metadata, 'n_chgset')
    metadata = ecdf_transform(metadata, 'n_total_chgset')
    return metadata

def ecdf_transform(metadata, feature):
    """ Apply an ECDF transform on feature within metadata; transform the column
    data into ECDF values

    Parameters
    ----------
    metadata: pd.DataFrame
        Metadata in which the transformation takes place
    feature: object
        string designing the transformed column

    Return
    ------
    New metadata, with a new renamed feature containing ecdf version of
    original data

    """
    ecdf = sm.distributions.ECDF(metadata[feature])
    metadata[feature] = ecdf(metadata[feature])
    new_feature_name = 'u_' + feature.split('_', 1)[1]
    return metadata.rename(columns={feature: new_feature_name})

def extract_generic_modif_features(metadata, data, grp_feat):
    """Extract generic features about modifications done by each individuals: number
    of modifications, number of modifications per element type

    Parameters
    ----------
    metadata: pd.DataFrame
        metadata table
    data: pd.DataFrame
        original data
    grp_feat: ojbect
        string designing the grouping feature: it characterizes the metadata
    ("chgset", or "user")

    """
    newfeature = (data.groupby([grp_feat])['id']
                  .count()
                  .reset_index()
                  .fillna(0))
    newfeature.columns = [grp_feat, "n_total_modif"]
    metadata = pd.merge(metadata, newfeature, on=grp_feat, how="outer").fillna(0)
    newfeature = (data.query('elem=="node"').groupby([grp_feat])['id']
                  .count()
                  .reset_index()
                  .fillna(0))
    newfeature.columns = [grp_feat, "n_total_modif_node"]
    metadata = pd.merge(metadata, newfeature, on=grp_feat, how="outer").fillna(0)
    newfeature = (data.query('elem=="way"').groupby([grp_feat])['id']
                  .count()
                  .reset_index()
                  .fillna(0))
    newfeature.columns = [grp_feat, "n_total_modif_way"]
    metadata = pd.merge(metadata, newfeature, on=grp_feat, how="outer").fillna(0)
    newfeature = (data.query('elem=="relation"').groupby([grp_feat])['id']
                  .count()
                  .reset_index()
                  .fillna(0))
    newfeature.columns = [grp_feat, "n_total_modif_relation"]
    metadata = pd.merge(metadata, newfeature, on=grp_feat, how="outer").fillna(0)

    return metadata

def extract_modif_features(metadata, data, element_type, grp_feat):
    """Extract a set of metadata features corresponding to a specific element
    type; centered on modifications

    Parameters
    ----------
    metadata: pd.DataFrame
        Metadata table
    data: pd.DataFrame
        Original data
    element_type: object
        string designing the element type ("node", "way", or "relation")
    grp_feat: object
        string designing the grouping feature; it characterizes the metadata
    ("chgset", or "user")

    """
    typed_data = data.query('elem==@element_type')
    metadata = create_count_features(metadata, element_type, typed_data,
                               grp_feat, 'id', '')
    metadata = create_count_features(metadata, element_type,
                               typed_data.query("init"),
                               grp_feat, 'id', "_cr")
    metadata = create_count_features(metadata, element_type,
                               typed_data.query("not init and visible"),
                               grp_feat, 'id', "_imp")
    metadata = create_count_features(metadata, element_type,
                               typed_data.query("not init and not visible"),
                               grp_feat, 'id', "_del")
    metadata = create_count_features(metadata, element_type,
                               typed_data.query("up_to_date"),
                               grp_feat, 'id', "_utd")
    metadata = create_count_features(metadata, element_type,
                               typed_data.query("willbe_corr"),
                               grp_feat, 'id', "_cor")
    metadata = create_count_features(metadata, element_type,
                               typed_data.query("willbe_autocorr"),
                               grp_feat, 'id', "_autocor")
    return metadata

def create_count_features(metadata, element_type, data, grp_feat, res_feat, feature_suffix):
    """Create an additional feature to metadata by counting number of
    occurrences in data, for a specific element_type

    Parameters
    ----------
    metadata: pd.DataFrame
        Metadata table
    element_type: object
        string designing the element type ("node", "way", or "relation")
    data: pd.DataFrame
        Original data
    grp_feat: object
        string designing the grouping feature; it characterizes the metadata
    ("chgset", or "user")
    res_feat: object
        string that indicates the measured feature (how many items correspond
    feature_suffix: type
        description

    """
    feature_name = 'n_'+ element_type + '_modif' + feature_suffix
    newfeature = (data.groupby([grp_feat])[res_feat]
                  .count()
                  .reset_index()
                  .fillna(0))
    newfeature.columns = [grp_feat, feature_name]
    metadata = pd.merge(metadata, newfeature, on=grp_feat, how="outer").fillna(0)
    return metadata

def extract_element_features(metadata, data, element_type, grp_feat):
    """Extract a set of metadata features corresponding to a specific element
    type; centered on unique elements

    Parameters
    ----------
    metadata: pd.DataFrame
        Metadata table
    data: pd.DataFrame
        Original data
    element_type: object
        string designing the element type ("node", "way", or "relation")
    grp_feat: object
        string designing the grouping feature; it characterizes the metadata
    ("chgset", or "user")

    """
    typed_data = data.query('elem==@element_type')
    metadata = create_unique_features(metadata, element_type,
                               typed_data.query("created and open"),
                                      grp_feat, 'id', "_cr")
    metadata = create_unique_features(metadata, element_type,
                               typed_data.query("created and open and not available"),
                               grp_feat, 'id', "_crwrong")
    normalize_features(metadata, 'n_'+element_type+'_cr')
    metadata = create_unique_features(metadata, element_type,
                               typed_data.query("not created and open"),
                               grp_feat, 'id', "_imp")
    metadata = create_unique_features(metadata, element_type,
                               typed_data.query("not created and open and not available"),
                               grp_feat, 'id', "_impwrong")
    normalize_features(metadata, 'n_'+element_type+'_imp')
    metadata = create_unique_features(metadata, element_type,
                               typed_data.query("not created and not open"),
                               grp_feat, 'id', "_del")
    metadata = create_unique_features(metadata, element_type,
                               typed_data.query("not created and not open and available"),
                               grp_feat, 'id', "_delwrong")
    normalize_features(metadata, 'n_'+element_type+'_del')
    return metadata

def create_unique_features(metadata, element_type, data, grp_feat, res_feat, feature_suffix):
    """Create an additional feature to metadata by counting number of unique
    occurrences in data, for a specific element_type

    Parameters
    ----------
    metadata: pd.DataFrame
        Metadata table
    element_type: object
        string designing the element type ("node", "way", or "relation")
    data: pd.DataFrame
        Original data
    grp_feat: object
        string designing the grouping feature; it characterizes the metadata
    ("chgset", or "user")
    res_feat: object
        string that indicates the measured feature (how many items correspond
    feature_suffix: type
        description

    """
    feature_name = 'n_'+ element_type + feature_suffix
    newfeature = (data.groupby([grp_feat])[res_feat]
                  .nunique()
                  .reset_index()
                  .fillna(0))
    newfeature.columns = [grp_feat, feature_name]
    metadata = pd.merge(metadata, newfeature, on=grp_feat, how="outer").fillna(0)
    return metadata

def extract_features(data, pattern, copy=True):
    """Extract features from data that respect the given string pattern

    Parameters
    ----------
    data: pd.DataFrame
    starting dataframe
    pattern: str
    character string that indicates which column has to be kept
    copy: boolean
    True if a copy of the data has to be returned, false otherwise
    """
    if copy:
        return data[[col for col in data.columns
                     if re.search(pattern, col) is not None]].copy()
    else:
        return data[[col for col in data.columns
                     if re.search(pattern, col) is not None]]

def drop_features(data, pattern, copy=True):
    """Drop features from data that respect the given string pattern

    Parameters
    ----------
    data: pd.DataFrame
    starting dataframe
    pattern: str
    character string that indicates which column has to be dropped
    copy: boolean
    True if a copy of the data has to be returned, false otherwise
    """
    if copy:
        return data[[col for col in data.columns
                     if re.search(pattern, col) is None]].copy()
    else:
        return data[[col for col in data.columns
                     if re.search(pattern, col) is None]]

def normalize_temporal_features(metadata, max_lifespan, timehorizon,
                                duration_feats=['lifespan',
                                               'n_inscription_days',
                                               'n_activity_days']):
    """Transform metadata features that are linked with temporal information

    Parameters
    ----------
    metadata: pd.DataFrame
        Metadata table, must contains temporal features
    timehorizon: pd.TimeDelta
        time horizon, used to normalize the temporal feature
    duration_feats: list of objects
        strings designing the name of the individuals activity duration

    """
    metadata[duration_feats[0]] = metadata[duration_feats[0]] / max_lifespan
    metadata[duration_feats[1]] = metadata[duration_feats[1]] / timehorizon
    metadata = ecdf_transform(metadata, duration_feats[2])

def normalize_features(metadata, total_column):
    """Transform values of metadata located in cols columns into percentages of
    values in total_column column, without attempting to metadata structure

    Parameters
    ----------
    metadata: pd.DataFrame
        Metadata table
    total_column: object
        String designing the reference column

    """
    transformed_columns = metadata.columns[metadata.columns.to_series()
                                           .str.contains(total_column)]
    metadata[transformed_columns[1:]] = metadata[transformed_columns].apply(lambda x: (x[1:]/x[0]).fillna(0), axis=1)

def logtransform_feature(metadata, column):
    """Apply a logarithm transformation to the column within
    metadata with; to avoid NaN, the applied operation is f:x->log(1+x)

    Parameters
    ----------
    metadata: pd.DataFrame
        Metadata
    column: object
        string designing the name of the column to transform

    """
    metadata[column] = metadata[column].apply(lambda x: math.log(1+x))


#!/usr/bin/env python
# coding: utf-8

import streamlit as st
import pandas as pd
import numpy as np
import plotly.express as px
import datetime
import matplotlib.pyplot as plt
import pickle
import pandas as pd
import numpy as np
import seaborn as sns
import re
import unidecode
import nltk
import spacy
import string
import matplotlib.pyplot as plt
import emoji
from TextCleaner import clean_text
from os import path
from PIL import Image
from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator

def predict_model(x, vectorizer, model):
    x = clean_text(x)

    x_tok = [y.lemma_ for y in  nlp(x)]

    #print(x_tok)

    x =  ' '.join(word for word in x_tok)

    #print(x)

    X_single=vectorizer.transform([x]);

    return model.predict(X_single)

happy_comment = "Muchas gracias por su comentario, en Balanz trabajamos cada dia para darle una mejor experiencia de usuario."
angry_comment =  "Sentimos mucho que haya tenido inconvenientes, a la brevedad lo contactaremos para solucionar el problema."

nlp = spacy.load("es_core_news_md")

vect_filename = 'saved_vect.sav'
model_filename = 'saved_model.sav'

# load the model from disk
loaded_vect = pickle.load(open(vect_filename, 'rb'))
loaded_model = pickle.load(open(model_filename, 'rb'))

st.set_page_config(page_title="Sentimiento", page_icon=":smile:", layout="wide")

# st.image('https://camfintech.hiringroomcampus.com/assets/media/camfintech/company_5f695d4a4b8e561c835eae32.jpg',width = 400)
st.image("https://www.digitalhouse.com/ar/logo-DH.png", width = 500)
st.title('Predictor de sentimiento')

st.text('Predicción por lote:')

uploaded_file = st.file_uploader("Elija un archivo de comentarios")
if uploaded_file is not None:
    df_comments = pd.read_csv(uploaded_file,sep=';')
    try:
        df_comments = df_comments.drop(['Subject','Date Created', "Sender's Name", 'Unread'], axis=1)
    except:
        df_comments = df_comments.drop(['Subject','Date Created', "Sender's Name"], axis=1)

    df_comments.Body = df_comments.Body.apply(lambda x:  x.split('\n')[-3])
    df_comments.Body = df_comments.Body.replace("\r", '')
    st.write(df_comments)

    df_comments['score'] = None
    df_comments['output_comment'] = None
    df_comments['emoji'] = None
    df_comments['Sentiment'] = None
    #s['icons'] = s['emoji'].apply(lambda x: emoji.emojize(x, use_aliases=True))
    i = 0
    for x in df_comments.Body:
        df_comments.at[i, 'score'] = predict_model(x, loaded_vect, loaded_model)
        i += 1

    df_comments.score = df_comments.score.astype(int)
    i = 0
    for x in df_comments.Body:

        if df_comments.loc[i, 'score']:
            df_comments.at[i, 'output_comment'] = happy_comment
            df_comments.at[i, 'emoji'] = ":heart_eyes:"
            df_comments.at[i, 'Sentiment'] = emoji.emojize(":heart_eyes:", use_aliases=True)
        else:
            df_comments.at[i, 'output_comment'] = angry_comment
            df_comments.at[i, 'emoji'] = ":cry:"
            df_comments.at[i, 'Sentiment'] = emoji.emojize(":cry:", use_aliases=True)

        i += 1

    df_comments = df_comments.drop(['emoji'], axis=1)

    st.table(df_comments)

    fig = px.pie(df_comments, names='Sentiment', title= "Inner App's reviews", width=800, height=400, color_discrete_sequence=["blue", "green"])
    fig.update_traces(textinfo="percent", insidetextfont= dict(color="white"))
    st.write(fig)

    #WordCloud de df_comments.Body
    text=' '.join(df_comments.Body.replace(' ','_').to_list())
    stop_words_custom = ['balanz','q', 'hola', 'má' 'tu', 'mis', 'brubank', 'etoro', 'broker', 'asesor', 'android', 'ios', 'y', 'a', 'de', 'muy', 'la', 'que', 'app', 'para', 'el', 'en', 'es', 'me', 'yo', 'he', 'se', 'con', 'los', 'un', 'las', 'pero', '1000', 'por', 'una','lo', 'todo', 'aplicacion', '12', '15', '24', '48', '48hs', 'mi', 'como', 'al','del', 'le', 'les', 'si','sin', 'ya', 'ahora', 'te', 'desde','balanz', 'hace', 'cuando', 'poder', 'bastante','puedo', 'mas', 'cada','esta', 'hay']
    wordcloud = WordCloud(width=800, height=400, max_font_size=200, max_words=50, background_color="white", stopwords=stop_words_custom).generate(text)

    # Display the generated image:
    fig = plt.figure(figsize=(15,10))
    plt.imshow(wordcloud)
    plt.axis("off")
    st.write(fig)
#     plt.show()

st.text('Prediga su comentario:')
default_value_goes_here = ""
user_input = st.text_input("Ingrese su comentario", default_value_goes_here)
if st.button('enviar') and (user_input is not None):
    score_num = predict_model(user_input, loaded_vect, loaded_model)
    score_num = score_num.astype(int)
    if score_num:
        st.markdown(':heart_eyes:')
        answer = happy_comment
    else:
        st.markdown(":cry:")
        answer = angry_comment

    st.write(answer)

# In[ ]:

hide_streamlit_style = """
            <style>
            #MainMenu {visibility: hidden;}
            footer {visibility: hidden;}
            footer:after {
                content:'Grupo 4 - Data Science - Digital House';
                visibility: visible;
                display: block;
                position: relative;
                #background-color: red;
                padding: 5px;
                top: 2px;
                font-size: 20px;
                color: green;
            }
            </style>
            """
st.markdown(hide_streamlit_style, unsafe_allow_html=True)


#! /usr/bin/env python3

from Bio import SeqIO
from argparse import ArgumentParser

parser = ArgumentParser(description='Filter FASTA file by the percentage of X and lowercase letters')
parser.add_argument('-f', type=str, help='FASTA file')
parser.add_argument('-n', type=float, help='Max acceptable proportion of X and lowercase')
args = parser.parse_args()

def lowercase_count(line):
    """
    A count of lowercase letters and/or X'es in a string
    :param line:
    :return:
    """
    count = 0
    for letter in line:
        if letter.islower() or letter == 'X':
            count += 1
    return count

with open('{}.filtered'.format(args.f), mode='w') as outfasta:
    for record in SeqIO.parse(args.f, 'fasta'):
        if lowercase_count(record.seq)/len(record) <= args.n:
            SeqIO.write(record, outfasta, 'fasta')


# coding=utf-8

from flask import jsonify, request, g, url_for
from .. import db
from ..models.models import Post, Permission
from ..models.manager import PostManager, UserManager
from . import api
from .decorators import permission_required
from .errors import forbidden
from .authentication import auth

@api.route('/posts/')
@auth.login_required
def get_posts():
    page = request.args.get('page', 1, type=int)
    pagination = Post.query.paginate(page, per_page=10, error_out=False)
    posts = pagination.items
    prev = None
    if pagination.has_prev:
        prev = url_for('api.get_posts', page=page-1, _external=True)
    next = None
    if pagination.has_next:
        next = url_for('api.get_posts', page=page+1, _external=True)
    return jsonify({
        'posts': [PostManager.to_json(post) for post in posts],
        'prev': prev,
        'next': next,
        'count': pagination.total
    })

@api.route('/posts/<int:id>')
@auth.login_required
def get_post(id):
    post = Post.query.get_or_404(id)
    return jsonify(PostManager.to_json(post))

@api.route('/posts/', methods=['POST'])
@permission_required(Permission.WRITE_ARTICLES)
def new_post():
    PostManager.add_post(head=request.form.get('head'), body=request.form.get('body'),
                         author=g.current_user)
    return jsonify({'success': True})

@api.route('/posts/<int:id>', methods=['POST'])
@permission_required(Permission.WRITE_ARTICLES)
def edit_post(id):
    post = Post.query.get_or_404(id)
    if g.current_user != post.author and \
            not UserManager.can(g.current_user, Permission.ADMINISTER):
        return forbidden('Insufficient permissions')
    post.title = request.form.get('head', post.head)
    post.body = request.form.get('body', post.body)
    db.session.add(post)
    return jsonify(PostManager.to_json(post))


from datetime import datetime, date
from typing import FrozenSet, List, Dict

INT_LENGTH = 9
STR_LENGTH = 9

INT_LIST_LENGTH = 5

MIN_YEAR = 0
MAX_YEAR = 2200
MIN_MONTH = 1
MAX_MONTH = 12
MIN_DAY = 1
MAX_DAY = 31
MIN_HOUR = 0
MAX_HOUR = 23
MIN_MINUTE = 0
MAX_MINUTE = 59
MIN_SECOND = 1
MAX_SECOND = 59

def supported_types() -> FrozenSet:
    return frozenset([int, bool, str, datetime, date, List[int], Dict[str, int], dict])


import tempfile
import unittest

from pyspark import Row

from sourced.ml.models import DocumentFrequencies
from sourced.ml.tests import create_spark_for_test
from sourced.ml.transformers import Indexer

class IndexerTests(unittest.TestCase):
    def setUp(self):
        data = [Row(to_index="to_index%d" % i, value=i) for i in range(10)]
        self.data = data
        self.session = create_spark_for_test()
        self.data_rdd = self.session.sparkContext \
            .parallelize(range(len(data))) \
            .map(lambda x: data[x])

    def test_call(self):
        indexer = Indexer("to_index")
        res = indexer(self.data_rdd)
        values = indexer.values()
        data_reverse = res \
            .map(lambda x: Row(to_index=values[x.to_index], value=x.value)) \
            .collect()
        self.assertEqual(self.data, data_reverse)

    def test_save_load(self):
        indexer = Indexer("to_index")
        res = indexer(self.data_rdd)
        with tempfile.NamedTemporaryFile(suffix="-index.asdf") as tmp:
            cached_index_path = tmp.name
            indexer.save_index(cached_index_path)
            docfreq = DocumentFrequencies().load(source=cached_index_path)
            document_index = {key: int(val) for (key, val) in docfreq}
            indexer = Indexer("to_index", column2id=document_index)
            self.assertEqual(res.collect(), indexer(self.data_rdd).collect())

if __name__ == "__main__":
    unittest.main()


from matplotlib import pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import numpy as np
import MyHelper
import FileOperator

f = open(".\FeatureVectors.txt",'a')
f.seek(0)
f.truncate()
f.close()

print("-----trees-----")
tree_array=np.zeros((10,2))#创建
for i in range(1,11):

    p= np.loadtxt("./points/new_trees/tree"+str(i)+".txt")
    print('No.',str(i),MyHelper.GetFeatureVector(p))
    v = MyHelper.GetFeatureVector(p)
    tree_array[i-1,:]=v[0,:]
    tree=plt.scatter(v[0,0],v[0,1],c = 'g')

    FileOperator.WriteData(".\FeatureVectors.txt",v,1)

print('tree_array.shape: ',tree_array.shape)

print("-----buildings-----")
building_array=np.zeros((10,2))
for i in range(1,11):
    p= np.loadtxt("./points/new_building/building"+str(i)+".txt")
    print('No.',str(i),MyHelper.GetFeatureVector(p))
    v = MyHelper.GetFeatureVector(p)
    tree_array[i-1, :] = v[0,:]
    # print(tree_array[i-1,:])
    building=plt.scatter(v[0,0],v[0,1],c = 'r')
    FileOperator.WriteData(".\FeatureVectors.txt",v,-1)

print('building_array.shape: ',building_array.shape)
plt.legend([tree,building],['tree','building'],)

#绘制
# plt.figure()
# tree=plt.scatter(tree_array[:,0],tree_array[:,1],c='g')
# building=plt.scatter(building_array[:,0],building_array[:,1],c='r')
# plt.legend([tree,building],['tree','building'])

plt.show()


#!/usr/bin/env python
# encoding: utf-8
#@contact: 罗湘飞
#@time: 2019/4/9 17:17

import requests
import json
from common.MD5page import md5Page
from common.data_config import PROT_YAML_DIR
from common.getYaml import get_yaml
class K8s_verify_code(object):
    '''发送短信接口'''
    def __init__(self):
        self.m = md5Page()
        self.yaml = get_yaml(PROT_YAML_DIR)
        self.time = self.m.time_str()
        #self.time = "20190404095149"
        self.data = '{"appid":"e338fb6e73154efeafbf5f71e77611a0","code":"161620","phone":"13694245189"}'.replace(' ','')
        #self.data = json.dumps({"appid":self.yaml["App-Id"],"phone":"13662460070","templateId":self.yaml["templateId"]})
    def __str__(self):
        pass
    def getSign(self):
        '''接口sign'''
        str1 = md5Page().strMD5(self.time)
        str2 = md5Page().strMD5(self.data)
        str3 = md5Page().strMD5("{}RCS_SIGN{}".format(str1, str2))
        return str3
    def verify_code_port(self):
        "发送短信接口请求"
        header={"time":self.time,
            "sign":self.getSign(),"Content - Type": "application/json",
        }
        data = json.loads(self.data)
        url=self.yaml["url"]["verify_code"]
        response = requests.post(url, headers=header, json=data)
        # res=json.loads(response.text)
        print(response.text)
        return response.text
if __name__ == '__main__':
    t=K8s_verify_code()
    t.verify_code_port()


import json as js

item = {
	"name" : "Johntavious",
	"age" : "22",
	"classes" : ["Junior Design", "Algorithms", "Prob/Stat", "Comp Sys Arch"],
	"Institution" : "Georgia Tech",
	"players_rank" : {
			1 : "Kyrie",
			2 : "Lebron",
			3 : "PG-13",
			4 : "Kobe"
		}
}
# source = open('data.json', 'r')
# json_obj = js.load(source)

with open("data.json", "w") as f:
	js.dump(item, f, indent=4);

# print(json_obj["name"])
# players = json_obj["players_rank"] #returns a map

# print(players);


"""
PatchCore logic based on https://github.com/rvorias/ind_knn_ad
"""

from sklearn import random_projection
from utils.utils import KNNGaussianBlur
from utils.utils import set_seeds
import numpy as np
from sklearn.metrics import roc_auc_score
import timm
import torch
from tqdm import tqdm
from utils.au_pro_util import calculate_au_pro

class Features(torch.nn.Module):

    def __init__(self, image_size=224, f_coreset=0.1, coreset_eps=0.9):
        super().__init__()
        self.device = "cuda" if torch.cuda.is_available() else "cpu"
        self.deep_feature_extractor = Model(device=self.device)
        self.deep_feature_extractor.to(self.device)
        self.deep_feature_extractor.freeze_parameters(layers=[], freeze_bn=True)

        self.image_size = image_size
        self.f_coreset = f_coreset
        self.coreset_eps = coreset_eps
        self.average = torch.nn.AvgPool2d(3, stride=1)
        self.blur = KNNGaussianBlur(4)
        self.n_reweight = 3
        set_seeds(0)
        self.patch_lib = []
        self.resize = torch.nn.AdaptiveAvgPool2d((28, 28))

        self.image_preds = list()
        self.image_labels = list()
        self.pixel_preds = list()
        self.pixel_labels = list()
        self.gts = []
        self.predictions = []
        self.image_rocauc = 0
        self.pixel_rocauc = 0
        self.au_pro = 0

    def __call__(self, x):
        # Extract the desired feature maps using the backbone model.
        with torch.no_grad():
            feature_maps = self.deep_feature_extractor(x)

        feature_maps = [fmap.to("cpu") for fmap in feature_maps]
        return feature_maps

    def add_sample_to_mem_bank(self, sample):
        raise NotImplementedError

    def predict(self, sample, mask, label):
        raise NotImplementedError

    def compute_s_s_map(self, patch, feature_map_dims, mask, label):
        dist = torch.cdist(patch, self.patch_lib)
        min_val, min_idx = torch.min(dist, dim=1)
        s_idx = torch.argmax(min_val)
        s_star = torch.max(min_val)

        # reweighting
        m_test = patch[s_idx].unsqueeze(0)  # anomalous patch
        m_star = self.patch_lib[min_idx[s_idx]].unsqueeze(0)  # closest neighbour
        w_dist = torch.cdist(m_star, self.patch_lib)  # find knn to m_star pt.1
        _, nn_idx = torch.topk(w_dist, k=self.n_reweight, largest=False)  # pt.2
        # equation 7 from the paper
        m_star_knn = torch.linalg.norm(m_test - self.patch_lib[nn_idx[0, 1:]], dim=1)
        # Softmax normalization trick as in transformers.
        # As the patch vectors grow larger, their norm might differ a lot.
        # exp(norm) can give infinities.
        D = torch.sqrt(torch.tensor(patch.shape[1]))
        w = 1 - (torch.exp(s_star / D) / (torch.sum(torch.exp(m_star_knn / D))))
        s = w * s_star

        # segmentation map
        s_map = min_val.view(1, 1, *feature_map_dims)
        s_map = torch.nn.functional.interpolate(s_map, size=(self.image_size, self.image_size), mode='bilinear')
        s_map = self.blur(s_map)

        self.image_preds.append(s.numpy())
        self.image_labels.append(label)
        self.pixel_preds.extend(s_map.flatten().numpy())
        self.pixel_labels.extend(mask.flatten().numpy())
        self.predictions.append(s_map.detach().cpu().squeeze().numpy())
        self.gts.append(mask.detach().cpu().squeeze().numpy())

    def calculate_metrics(self):
        self.image_preds = np.stack(self.image_preds)
        self.image_labels = np.stack(self.image_labels)
        self.pixel_preds = np.array(self.pixel_preds)

        self.image_rocauc = roc_auc_score(self.image_labels, self.image_preds)
        self.pixel_rocauc = roc_auc_score(self.pixel_labels, self.pixel_preds)
        self.au_pro, _ = calculate_au_pro(self.gts, self.predictions)

    def run_coreset(self):
        self.patch_lib = torch.cat(self.patch_lib, 0)
        if self.f_coreset < 1:
            self.coreset_idx = self.get_coreset_idx_randomp(self.patch_lib,
                                                            n=int(self.f_coreset * self.patch_lib.shape[0]),
                                                            eps=self.coreset_eps, )
            self.patch_lib = self.patch_lib[self.coreset_idx]

    def get_coreset_idx_randomp(self, z_lib, n=1000, eps=0.90, float16=True, force_cpu=False):
        """Returns n coreset idx for given z_lib.
        Performance on AMD3700, 32GB RAM, RTX3080 (10GB):
        CPU: 40-60 it/s, GPU: 500+ it/s (float32), 1500+ it/s (float16)
        Args:
            z_lib:      (n, d) tensor of patches.
            n:          Number of patches to select.
            eps:        Agression of the sparse random projection.
            float16:    Cast all to float16, saves memory and is a bit faster (on GPU).
            force_cpu:  Force cpu, useful in case of GPU OOM.
        Returns:
            coreset indices
        """

        print(f"   Fitting random projections. Start dim = {z_lib.shape}.")
        try:
            transformer = random_projection.SparseRandomProjection(eps=eps)
            z_lib = torch.tensor(transformer.fit_transform(z_lib))
            print(f"   DONE.                 Transformed dim = {z_lib.shape}.")
        except ValueError:
            print("   Error: could not project vectors. Please increase `eps`.")

        select_idx = 0
        last_item = z_lib[select_idx:select_idx + 1]
        coreset_idx = [torch.tensor(select_idx)]
        min_distances = torch.linalg.norm(z_lib - last_item, dim=1, keepdims=True)
        # The line below is not faster than linalg.norm, although i'm keeping it in for
        # future reference.
        # min_distances = torch.sum(torch.pow(z_lib-last_item, 2), dim=1, keepdims=True)

        if float16:
            last_item = last_item.half()
            z_lib = z_lib.half()
            min_distances = min_distances.half()
        if torch.cuda.is_available() and not force_cpu:
            last_item = last_item.to("cuda")
            z_lib = z_lib.to("cuda")
            min_distances = min_distances.to("cuda")

        for _ in tqdm(range(n - 1)):
            distances = torch.linalg.norm(z_lib - last_item, dim=1, keepdims=True)  # broadcasting step
            min_distances = torch.minimum(distances, min_distances)  # iterative step
            select_idx = torch.argmax(min_distances)  # selection step

            # bookkeeping
            last_item = z_lib[select_idx:select_idx + 1]
            min_distances[select_idx] = 0
            coreset_idx.append(select_idx.to("cpu"))
        return torch.stack(coreset_idx)

class Model(torch.nn.Module):

    def __init__(self, device, backbone_name='wide_resnet50_2', out_indices=(2, 3), checkpoint_path='',
                 pool_last=False):
        super().__init__()
        # Determine if to output features.
        kwargs = {'features_only': True if out_indices else False}
        if out_indices:
            kwargs.update({'out_indices': out_indices})

        self.backbone = timm.create_model(model_name=backbone_name, pretrained=True, checkpoint_path=checkpoint_path,
                                          **kwargs)
        self.device = device
        self.avg_pool = torch.nn.AdaptiveAvgPool2d((1, 1)) if pool_last else None

    def forward(self, x):
        x = x.to(self.device)

        # Backbone forward pass.
        features = self.backbone(x)

        # Adaptive average pool over the last layer.
        if self.avg_pool:
            fmap = features[-1]
            fmap = self.avg_pool(fmap)
            fmap = torch.flatten(fmap, 1)
            features.append(fmap)

        return features

    def freeze_parameters(self, layers, freeze_bn=False):
        """ Freeze resent parameters. The layers which are not indicated in the layers list are freeze. """

        layers = [str(layer) for layer in layers]
        # Freeze first block.
        if '1' not in layers:
            if hasattr(self.backbone, 'conv1'):
                for p in self.backbone.conv1.parameters():
                    p.requires_grad = False
            if hasattr(self.backbone, 'bn1'):
                for p in self.backbone.bn1.parameters():
                    p.requires_grad = False
            if hasattr(self.backbone, 'layer1'):
                for p in self.backbone.layer1.parameters():
                    p.requires_grad = False

        # Freeze second block.
        if '2' not in layers:
            if hasattr(self.backbone, 'layer2'):
                for p in self.backbone.layer2.parameters():
                    p.requires_grad = False

        # Freeze third block.
        if '3' not in layers:
            if hasattr(self.backbone, 'layer3'):
                for p in self.backbone.layer3.parameters():
                    p.requires_grad = False

        # Freeze fourth block.
        if '4' not in layers:
            if hasattr(self.backbone, 'layer4'):
                for p in self.backbone.layer4.parameters():
                    p.requires_grad = False

        # Freeze last FC layer.
        if '-1' not in layers:
            if hasattr(self.backbone, 'fc'):
                for p in self.backbone.fc.parameters():
                    p.requires_grad = False

        if freeze_bn:
            for module in self.backbone.modules():
                if isinstance(module, torch.nn.modules.BatchNorm1d):
                    module.eval()
                if isinstance(module, torch.nn.modules.BatchNorm2d):
                    module.eval()
                if isinstance(module, torch.nn.modules.BatchNorm3d):
                    module.eval()


from pdf2image import convert_from_path
import os
from PIL import Image
import logging
import concurrent.futures

# Set up logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

def convert_pdf_to_image(pdf_path, output_folder, output_size=(1920, 2560)): # attention
    # Check if the file is a PDF (case-insensitive)
    if pdf_path.lower().endswith('.pdf'):
        try:
            # Convert the first page of the PDF to an image
            images = convert_from_path(pdf_path, first_page=1, last_page=1)

            # Save the image
            for i, image in enumerate(images):
                # Create the output file path
                base_name = os.path.basename(pdf_path)
                name_without_ext = os.path.splitext(base_name)[0]
                image_path = os.path.join(output_folder, f"{name_without_ext}.jpg")

                # Resize the image
                image.thumbnail(output_size, Image.LANCZOS)

                # Save the image
                image.save(image_path, 'JPEG')

            logger.info(f'Successfully converted {pdf_path} to image and saved at {image_path}')

        except Exception as e:
            logger.error(f'Failed to convert {pdf_path} to image: {e}')

def parallel_convert_pdf_to_image(pdf_folder, output_folder, output_size=(1920, 2560)): # attention
    # Create 'images' directory if it does not exist
    if not os.path.exists(output_folder):
        os.makedirs(output_folder)

    # Get all PDF files in the folder
    pdf_files = [os.path.join(pdf_folder, filename) for filename in os.listdir(pdf_folder) if filename.lower().endswith('.pdf')]

    # Use a ThreadPoolExecutor to parallelize the conversion process
    with concurrent.futures.ThreadPoolExecutor() as executor:
        executor.map(convert_pdf_to_image, pdf_files, [output_folder]*len(pdf_files), [output_size]*len(pdf_files))

# Usage
parallel_convert_pdf_to_image('./pdfs_1077', './images_1920_2560') # attention

# 1280, 1600
# 1440, 1920
# 1920, 2560

# 1080, 1440
# 1140, 1520
# 1260, 1680
# 1320, 1760
# 1380, 1840
# 1500, 2000
# 1560, 2080
# 1620, 2160
# 1680, 2240
# 1740, 2320
# 1800, 2400
# 1860, 2480


# -*- coding: utf-8 -*-
"""
Created on Mon Mar  8 19:38:25 2021

@author: shunyu
"""

from scipy.spatial.transform import Rotation
import numpy as np
import ipyvolume as ipv
import matplotlib.pyplot as plt
from numba import jit
from tqdm import tqdm

##### R is the half length of a nanocube edge
R = 5.15
#####################################
#  D is 2 times of lattice constant
#####################################
D = 45.6
#### p is the corresponding reciprocal frequency to create the periodic structure in
p = 2*np.pi/D

############## for a specific rotation #############
Rot_step = [[35.3, 30, 54.7]]

'''
############## for a series of ratation #############
Rot_step = []
for omiga in np.linspace(0, 90, num=9):
    for delta in np.linspace(0, 90, num=9):
        for theta in np.linspace(0, 90, num=9):
            Rot_step.append([omiga, delta, theta])
'''

Vector = np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]])
RVector = []
for Rot in Rot_step:
    r = Rotation.from_euler('xyz', Rot, degrees=True)
    rVector = r.apply(Vector)
    RVector.append(rVector)

########### define reciprocal space ################

### defube wavevector range
axis = np.linspace(-3,3,301)

### constructe 3D reciprocal space
x, y, z = np.meshgrid(axis, axis, axis)

### calculate wavevector in spherical symmetry
q = np.sqrt(x**2+y**2+z**2)

### get 1D wavevector
hist, bin_edges = np.histogram(q, bins= 120, range = (0.1, 2.75))
qAxis = (bin_edges[1:] + bin_edges[:-1]) / 2.0

########### define form factor and structure factor ################
### define function to get the form factor of a cube
@jit(nopython=True)
def ff(scale, X, Y, Z, R):
    return scale*(8*np.sinc((X)*R/np.pi)*np.sinc((Y)*R/np.pi)*np.sinc((Z)*R/np.pi))**2

### define the Lorentzian function broadened FCC packing, which is a BCC in reciprocal space
@jit(nopython=True)
def fcc(p, o):
    return 4/(1+(np.abs(((x+y)/2)%(2*p)-p)-p)**2/o**2)/(1+(np.abs(((y+z)/2)%(2*p)-p)-p)**2/o**2)/(1+(np.abs(((x+z)/2)%(2*p)-p)-p)**2/o**2)

### calculate the structure factor of FCC
s_fcc = fcc(p, 0.01)

### create an array to collect all 1D Scattering profile of FCC-nanocube system
A_fcc = []

'''
### define the Lorentzian function broadened BCC packing, which is a FCC in reciprocal space
@jit(nopython=True)
def bcc(scale, X, Y, Z, R, p, o):
    return 2/(1+(np.abs(((x+y+z)/2)%(2*p)-p)-p)**2/o**2)/(1+(np.abs(((x-y+z)/2)%(2*p)-p)-p)**2/o**2)/(1+(np.abs(((-x+y+z)/2)%(2*p)-p)-p)**2/o**2)

### calculate the structure factor of BCC
s_bcc = bcc(p, 0.01)

### create an array to collect all 1D Scattering profile of BCC-nanocube system
A_bcc = []

### define the Lorentzian function broadened SC packing, which is a SC in reciprocal space
@jit(nopython=True)
def sc(p, o):
    return 1/(1+(np.abs((x)%(2*p)-p)-p)**2/o**2)/(1+(np.abs((y)%(2*p)-p)-p)**2/o**2)/(1+(np.abs((z)%(2*p)-p)-p)**2/o**2)

### calculate the structure factor of SC
s_sc = sc(p, 0.02)

### create an array to collect all 1D Scattering profile of FCC-nanocube system
A_sc = []

'''
for counter, vector_new in enumerate(tqdm(RVector)):

    X = vector_new[0][0]*x+vector_new[0][1]*y+vector_new[0][2]*z
    Y = vector_new[1][0]*x+vector_new[1][1]*y+vector_new[1][2]*z
    Z = vector_new[2][0]*x+vector_new[2][1]*y+vector_new[2][2]*z

    f_fcc = ff(1, X, Y, Z, R)*s_fcc
    #f_sc = ff(1, X, Y, Z, R)*s_sc
    #f_bcc = ff(1, R, 2*np.pi/35, 0.02)

    r_fcc, bin_number = np.histogram(q, bins = 120, weights=f_fcc, range = (0.1, 2.75))
    r_fcc = r_fcc/hist
    A_fcc.append(r_fcc)
    #r_sc, bin_number = np.histogram(q, bins = 120, weights=f_sc, range = (0.1, 2.75))
    #r_sc = r_sc/hist
    #A_sc.append(r_sc)
    #r_bcc, bin_number = np.histogram(q, bins = 120, weights=f_bcc, range = (0.1, 2.75))
    #r_bcc = r_bcc/hist
    #A_bcc.append(r_bcc)


import pyautogui
import time
import random

# time.sleep(5)
# pyautogui.press("down")
def uparrow():
    time.sleep(5)
    pyautogui.press("up")

def downarrow():
    time.sleep(5)
    pyautogui.press("Down")

def like():
    time.sleep(5)
    # pyautogui.press("l")
    start = pyautogui.locateCenterOnScreen('liked.png')#If the file is not a png file it will not work
    # print(start)
    if start:
        print("already Liked")
    else:
        print("Bot liked")
        pyautogui.press("l")

# like()
commentlist = ["🏆🏆🏆", "king 👑", "Drop it 🔥" ,
                "Wow " , "Dope" , "💪💪" ,"👍👍","Fireeee" , "My man👑" ,
                "❤❤❤" ,"Hell yeah" , "Lets goooooo ❤🔥❤🔥" ,
                "Wooo" , "Dope af" , "crazy af 💞",
                "won my heard", "Bro drop this asap",
                "Sad for those who havnt yet discovered this music",
                "Sin the best", "God level", "💪💪" , "🙏🙏🙏", "Weeeew 🙂" ,
                "Greateeee", "Real Barz" ,"Rap Gold", "This is some real ..." ,
                "Holly Molly"]

def writeComment():
    int  = random.randint(0,len(commentlist)-1)
    # print(int,":" ,commentlist[int])
    comment = commentlist[int]
    time.sleep(4)
    commentarea = pyautogui.locateCenterOnScreen("realcomment.png")
    pyautogui.moveTo(commentarea)
    print(commentarea)

    time.sleep(1)
    pyautogui.leftClick()
    time.sleep(1)
    pyautogui.typewrite(comment)
    pyautogui.press("enter")
    print("entered")

    # to get out of the comment section.

# change over here
# we can fetch the video count somehow. and store it in totalVideos Variable.
totalVideos = 53
def mainup():
    counter =0
    Up =True
    while Up:
        counter +=1
        if counter<totalVideos:
            uparrow()
        else:
            print("No More Up")
            up = False
            maindown()

def maindown():
    counter =0
    Down =True
    while Down:
        counter +=1
        like()
        # writeComment()
        # time.sleep(1)
        # pyautogui.press("tab")
        if counter<totalVideos:
            downarrow()
        else:
            print("No More down")
            mainup()
            Down = False

maindown()


from django.shortcuts import render, redirect
from django.contrib.auth.decorators import login_required
from .models import *
from django.contrib import messages
from django.core.paginator import Paginator
import json
from django.http import JsonResponse, HttpResponse
import datetime
from userpreferences.models import UserPreferences
import csv
from django.db.models import Sum
from django.db.models import Count
from django.template.loader import render_to_string
from weasyprint import HTML
import tempfile

# Create your views here.

def search_expenses(request):
	if request.method == 'POST':
		search_str = json.loads(request.body).get('searchText')
		expenses = Expense.objects.filter(amount__istartswith = search_str, owner = request.user) | Expense.objects.filter(date__istartswith = search_str, owner = request.user) | Expense.objects.filter(description__icontains = search_str, owner = request.user) | Expense.objects.filter(category__icontains = search_str, owner = request.user)
		data = expenses.values()
		return JsonResponse(list(data), safe = False)

@login_required(login_url = '/authentication/login')
def index(request):
	expense = Expense.objects.filter(owner = request.user)
	paginator = Paginator(expense, 4)
	page_number = request.GET.get('page')
	page_obj = Paginator.get_page(paginator, page_number)
	try:
		currency = UserPreferences.objects.get(user = request.user).currency
	except:
		currency = None

	total_price = Expense.objects.filter(owner = request.user).aggregate(total=Sum('amount'))
	context = {

	'expense' : expense,
	'page_obj' : page_obj,
	'currency' : currency,
	'total_price':total_price

	}

	return render(request, 'expenses/index.html', context)

def add_expense(request):

	categories = Category.objects.all()
	category_user = Category_User.objects.filter(owner = request.user)
	category_length = Category_User.objects.filter(owner = request.user).aggregate(num_cate=Count('name'))

	if category_length['num_cate'] == 0:
		messages.error(request, "You have not added categories yet, Add them.")
	else:
		pass

	context = {

	'categories' : categories,
	'values' : request.POST,
	'category_user' : category_user,

	}

	if request.method == 'GET':
		return render(request, 'expenses/add-expense.html', context)

	if request.method == 'POST':
		amount = request.POST['amount']

		if not amount:
			messages.error(request, "Oops, you have to enter amount")
			return render(request, 'expenses/add-expense.html', context)

		description = request.POST['description']
		date = request.POST['expense_date']
		category = request.POST['category']

		if not description:
			messages.error(request, "Oops, you have to enter description")
			return render(request, 'expenses/add-expense.html', context)

		Expense.objects.create(owner = request.user, amount=amount, category=category, date=date, description=description)
		messages.success(request, "Your expense is added successfully.")
		return redirect('expenses')

def expense_edit(request, id):

	expense = Expense.objects.get(pk = id)
	categories = Category.objects.all()
	category_user = Category_User.objects.filter(owner = request.user)

	context = {

			'expense' : expense,
			"values" : expense,
			'categories' : categories,
			'category_user' : category_user

			}

	if request.method == 'GET':
		return render(request, 'expenses/edit-expense.html', context)

	if request.method == 'POST':

		amount = request.POST['amount']

		if not amount:
			messages.error(request, "Oops, you have to enter amount")
			return render(request, 'expenses/edit-expense.html', context)

		description = request.POST['description']
		date = request.POST['expense_date']
		category = request.POST['category']

		if not description:
			messages.error(request, "Oops, you have to enter description")
			return render(request, 'expenses/edit-expense.html', context)

		expense.owner = request.user
		expense.amount = amount
		expense.category = category
		expense.date = date
		expense.description = description
		expense.save()

		messages.success(request, "Your expense is updated successfully.")
		return redirect('expenses')

def delete_expense(request, id):
	expense = Expense.objects.get(pk = id)
	expense.delete()
	messages.info(request, "We have successfully deleted your expense information.")
	return redirect("expenses")

def expense_category_summary(request):
	todays_date = datetime.date.today()
	six_months_ago = todays_date-datetime.timedelta(days=30*6)

	expenses = Expense.objects.filter(owner= request.user, date__gte = six_months_ago, date__lte = todays_date)

	finalrep = {}

	def get_category(expense):
		return expense.category

	category_list = list(set(map(get_category, expenses)))

	def get_expense_category_amount(category):
		amount = 0

		filtered_by_category = expenses.filter(category = category)

		for item in filtered_by_category:
			amount += item.amount

		return amount

	for x in expenses:
		for y in category_list:
			finalrep[y] = get_expense_category_amount(y)

	return JsonResponse({'expense_category_data': finalrep}, safe=False)

def stat_view(request):
	return render(request, 'expenses/stat.html')

def export_csv(request):
	response = HttpResponse(content_type = 'text/csv')
	response['Content-Disposition'] = 'attachement; filename=Expenses' + \
	str(datetime.datetime.now()) + '.csv'

	writer = csv.writer(response)

	writer.writerow(['Amount', 'Description', 'Category', 'Date'])

	expense = Expense.objects.filter(owner = request.user)

	for ex in expense:
		writer.writerow([ex.amount, ex.description, ex.category, ex.date])

	return response

def add_category(request):

	if request.method == 'POST':
		category = request.POST['username']

		if not category:
			messages.error(request, "You have to enter a category first.")
			return render(request, 'expenses/add-category.html')
		Category_User.objects.create(owner = request.user, name=category)
		messages.success(request, "Your category has been added successfully")

	return render(request, 'expenses/add-category.html')

def export_pdf(request):
	response = HttpResponse(content_type = 'application/pdf')

	response['Content-Disposition'] = 'attachement; filename=Expenses' + \
	str(datetime.datetime.now()) + '.pdf'

	response['Content-Transfer-Encoding'] = 'binary'

	html_string = render_to_string('expenses/pdf-output.html', {'expenses':[],'total':0})

	html = HTML(string=html_string)

	result = html.write_pdf()

	with tempfile.NamedTemporaryFile(delete = True) as output:
		output.write(result)
		output.flush()
		output = open(output.name, 'rb')
		response.write(output.read())
	return response


""" Implements discriminative loss for autoencoder. """
import torch
import torch.nn as nn

class DRAELossAutograd(nn.Module):

    def __init__(self, lamb, size_average=True):
        super(DRAELossAutograd, self).__init__()
        self.lamb = lamb
        self.size_average = size_average  # for compatibility, not used

    def forward(self, inputs, targets):
        err = inputs.sub(targets).pow(2).view(inputs.size(0), -1).sum(dim=1, keepdim=False)
        err_sorted, _ = torch.sort(err)
        total_scatter = err.sub(err.mean()).pow(2).sum()
        regul = 1e6
        obj = None
        for i in range(inputs.size(0)-1):
            err_in = err_sorted[:i+1]
            err_out = err_sorted[i+1:]
            within_scatter = err_in.sub(err_in.mean()).pow(2).sum() + \
                             err_out.sub(err_out.mean()).pow(2).sum()
            h = within_scatter.div(total_scatter)
            if h < regul:
                regul = h
                obj = err_in.mean()

        return obj + self.lamb * regul