from sys import exc_info
from traceback import format_exception
from os.path import isfile
from types import ModuleType
from functools import wraps
from typing import Any, List, Dict, Union, Callable, ForwardRef, Awaitable
from inspect import getmembers, signature, Signature, isfunction, isclass
from dataclasses import fields, Field
from pathlib import PurePath
from time import perf_counter, process_time
from asyncio import iscoroutinefunction
import re
from datetime import datetime
from mimetypes import guess_type
from email.encoders import encode_base64
from email.mime.multipart import MIMEMultipart
from email.mime.base import MIMEBase
from email.mime.text import MIMEText

from pymongo import ASCENDING
from pymongo.errors import DuplicateKeyError
from motor.motor_asyncio import AsyncIOMotorClient, AsyncIOMotorGridFSBucket

from dataclasses_jsonschema import JsonSchemaMixin, ValidationError

from aiosmtplib import send

from sanic import Sanic, response
from sanic.request import Request
from sanic.log import logger
from sanic.exceptions import abort, NotFound, Unauthorized

from yrest.tree import Tree
from yrest.mongo import MongoJSONEncoder, Mongo
from yrest.openapi import OpenApi
from yrest.utils import Ok, OkResult, OkListResult, Error, ErrorMessage
from yrest.auth import AuthToken

class yJSONEncoder(MongoJSONEncoder):
  def default(self, obj):
    if isclass(obj) or isinstance(obj, (Tree, Mongo)):
      return repr(obj)
    elif isinstance(obj, datetime):
      return obj.isoformat()
    elif isinstance(obj, set):
      return str(obj)
    elif isinstance(obj, (ForwardRef, Field)):
      return str(obj)
    else:
      return MongoJSONEncoder.default(self, obj)

def timed(func):
  @wraps(func)
  async def decorated(*args, **kwargs):
    counter, time = perf_counter(), process_time()

    code = 200
    result = await func(*args, **kwargs)
    if isinstance(result, AuthToken):
      result = result.to_dict()
    elif isinstance(result, (Ok, Error)):
      result = result.to_dict()
      code = result.pop("code")

    result["pref_counter"] = perf_counter() - counter
    result["process_time"] = process_time() - time

    return response.json(result, code)
  return decorated

class ySanic(Sanic):
  def __init__(self, root_model: Tree, models: ModuleType, **kwargs: Dict[str, Any]):
    super().__init__(**kwargs)

    self._root_model = root_model
    self._models = models

    self._introspection = {}
    tree = self._introspect(tree = [])
    # print("\n".join(tree))
    # from json import dumps
    # logger.info(dumps(self._introspection["Group"], indent = 2, cls = yJSONEncoder))

    self._build_routes()

  def _introspect(self, model: Tree = None, analized: List[str] = None, indent: int = 0, tree: List[str] = None):
    if model is None:
      model = self._root_model

    if analized is None:
      analized = []

    if tree is not None:
      tab = "|  " * indent
      tree.append(f"{tab}{model.__name__}")

    if model.__name__ not in analized:
      self._introspection[model.__name__] = self._analize(model)
      analized.append(model.__name__)

    factories = []
    for field in fields(model):
      if "model" in field.metadata:
        factories.append(field.metadata['model'])

        child = getattr(self._models, field.metadata["model"])
        if child != model:
          self._introspect(child, analized, indent + 1, tree)

    if factories:
      self._introspection[model.__name__]["factories"] = factories

    return tree

  def _analize(self, model: Tree) -> Dict[str, str]:
    result = {}

    for name, member in getmembers(model, lambda m: iscoroutinefunction(m) or isfunction(m)):
      sig = signature(member)
      if "request" in sig.parameters.keys():
        result["call" if name == "index" else name] = self._check_signature(model, name, member, sig)

    return result

  def _check_signature(self, model: Tree, name: str, member: Awaitable, sig: Signature) -> Dict[str, str]:
    model_name = model.__name__
    is_root = model == self._root_model
    is_recursive = getattr(model, "_is_recursive", False)

    result = {}

    if hasattr(member, "__doc__") and member.__doc__:
      result["description"] = member.__doc__

    for param_name, param in sig.parameters.items():
      if param_name == "actor":
        result["actor"] = True
      elif param_name == "consume":
        result["consumes"] = getattr(self._models, param.annotation) if isinstance(param.annotation, str) else param.annotation

        urls = []

        consumed = result["consumes"].__name__.lower()
        if name == f"create_{consumed}":
          result["verb"] = "POST"
          if is_root:
            urls.append(f"/new/{consumed}")
          if not is_root or is_recursive:
            urls.append(f"/{{{model_name}_Path}}/new/{consumed}")
        else:
          result["verb"] = "POST" if name == "auth" else "PUT"
          if is_root:
            urls.append(f"/{name}")
          if not is_root or is_recursive:
            urls.append(f"/{{{model_name}_Path}}/{name}")

        result["urls"] = urls

    if "verb" not in result:
      urls = []
      if name == "remove":
        result["verb"] = "DELETE"
        if is_root:
          urls.append("/")
        if not is_root or is_recursive:
          urls.append(f"/{{{model_name}_Path}}/")
      else:
        result["verb"] = "GET"
        if is_root:
            urls.append("/" if name == "index" else f"/{name}")
        if not is_root or is_recursive:
          urls.append(f"/{{{model_name}_Path}}/" if name == "index" else f"/{{{model_name}_Path}}/{name}")
      result["urls"] = urls

    if hasattr(member, "__decorators__"):
      if "can_crash" in member.__decorators__:
        result["can_crash"] = member.__decorators__["can_crash"]

    result["produces"] = sig.return_annotation.__args__ if getattr(sig.return_annotation, "__origin__", False) == Union else sig.return_annotation

    return result

  def _build_routes(self):
    if "call" in self._introspection[self._root_model.__name__]:
      self.add_route(self.dispatcher, "/", ["GET"])
      self.add_route(self._generic_options, "/", ["OPTIONS"])

    if "update" in self._introspection[self._root_model.__name__]:
      self.add_route(self.updater, "/", ["PUT"])

    if hasattr(self, 'ws_endpoint'):
      self.add_websocket_route(self.ws_endpoint, "/ws")

    if "auth" in self._introspection[self._root_model.__name__]:
      self.add_route(self.auth, "/auth", ["POST"])
      self.add_route(self._generic_options, "/auth", ["OPTIONS"])

    if "factories" in self._introspection[self._root_model.__name__]:
      self.add_route(self.factory, "/new/<model>", ["POST"])
      self.add_route(self._generic_options, "/new/<model>", ["OPTIONS"])

    if isinstance(self, OpenApi):
      self.add_route(self.openapi, "/openapi", ["GET"])
      self.add_route(self._generic_options, "/openapi", ["OPTIONS"])

    not_root_models = list(self._introspection.keys())
    not_root_models.pop(not_root_models.index(self._root_model.__name__))
    not_root_factories = list(filter(lambda m: "factories" in self._introspection[m], not_root_models))
    if getattr(self._root_model, "_is_recursive", False) or len(not_root_factories):
      self.add_route(self.factory, "/<path:path>/new/<model>", ["POST"])
      self.add_route(self._generic_options, "/<path:path>/new/<model>", ["OPTIONS"])

    self.add_route(self.dispatcher, "/<path:path>", ["GET"])
    self.add_route(self.updater, "/<path:path>", ["PUT"])
    self.add_route(self.remover, "/<path:path>", ["DELETE"])
    self.add_route(self._generic_options, "/<path:path>", ["OPTIONS"])

  async def get_path(self, url: str, models, tolerance: int = 0) -> Dict[str, Any]:
    if url == "/":
      return await self._root_model.get(self._table, path = "")

    url_ = PurePath(url)
    test = 0
    while url_ != url_.parent:
      doc = await self._root_model._get_doc(self._table, url = str(url_))
      if doc:
        paper = getattr(models, doc["type"])(**doc)
        return paper

      test += 1
      if test > tolerance:
        break

      url_ = url_.parent
      if str(url_) == "/":
        return await self._root_model.get(self._table, path = "")

    raise NotFound(f"{url} not found")

  @timed
  async def auth(self, request: Request):
    if request.json is None:
      return ErrorMessage(message = f"Data must be provided",  code = 400)

    root = await self._root_model.get(self._table, path = "")
    auth = self._models.Auth(**request.json)
    return await root.auth(request, auth)

  @timed
  async def updater(self, request: Request, path: str = None):
    if request.json is None:
      return ErrorMessage(message = f"Data must be provided",  code = 400)

    path_ = f"/{path or ''}"
    try:
      paper = await self.get_path(path_, self._models, 1)
      paper._table = request.app._table
    except NotFound as e:
      return ErrorMessage(message = e.args[0], code = 404)

    url = paper.get_url()
    member = path_.replace(url, "")[1:] if url > "/" else path_[1:]
    if not member:
      member = "update"

    perm = await self._models.Permission.get(self._table, context = paper.type, name = member)
    token = AuthToken.get(request.headers)
    actor = await token.get_actor(self._table, self.config["JWT_SECRET"], self._models.User)
    # actor = await self._models.User.get(self._table, slug = "garito")
    if not perm or not await perm.allows(actor, paper):
      return ErrorMessage(message = "Unauthorized", code = 401)

    args = [request]
    _introspection = self._introspection[paper.type]["call" if member == "index" else member]
    if "actor" in _introspection:
      args.append(actor)

    consumes = _introspection.get("consumes", None)
    try:
      model = consumes(**request.json)
      args.append(model)
    except TypeError as e:
      return ErrorMessage(message = f"Validation error: {e}", code = 400)

    try:
      result = await getattr(paper, member)(*args)
      if isinstance(result, Tree):
        result = result.to_plain_dict()
      return OkListResult(result = result, code = 200) if isinstance(result, list) else  OkResult(result = result, code = 200)
    except Exception as e:
      message = format_exception(*exc_info()) if request.app.config.get("DEBUG", False) else str(e)
      for line in message:
        logger.error(line)
      return ErrorMessage(message = message, code = 400)

  @timed
  async def dispatcher(self, request, path: str = None):
    path_ = f"/{path or ''}"
    try:
      paper = await self.get_path(path_, self._models, 1)
      paper._table = request.app._table
    except NotFound as e:
      return ErrorMessage(message = e.args[0], code = 404)

    url = paper.get_url()
    member = path_.replace(url, "")[1:] if url > "/" else path_[1:]
    if not member:
      member = "index"

    perm = await self._models.Permission.get(self._table, context = paper.type, name = "call" if member == "index" else member)
    token = AuthToken.get(request.headers)
    actor = await token.get_actor(self._table, self.config["JWT_SECRET"], self._models.User)
    # actor = await self._models.User.get(self._table, slug = "garito")
    if not perm or not await perm.allows(actor, paper):
      return ErrorMessage(message = "Unauthorized", code = 401)

    args = [request]

    _introspection = self._introspection[paper.type]["call" if member == "index" else member]
    if "actor" in _introspection:
      args.append(actor)

    try:
      result = await getattr(paper, member)(*args)
      if isinstance(result, Tree):
        result = result.to_plain_dict()
      return OkListResult(result = result, code = 200) if isinstance(result, list) else  OkResult(result = result, code = 200)
    except Exception as e:
      message = format_exception(*exc_info()) if request.app.config.get("DEBUG", False) else str(e)
      for line in message:
        logger.error(line)
      return ErrorMessage(message = message, code = 500)

  @timed
  async def factory(self, request, model, path: str = None):
    path_ = f"/{path or ''}"
    try:
      paper = await self.get_path(path_, self._models, 0)
      paper._table = request.app._table
    except NotFound as e:
      return ErrorMessage(message = e.args[0], code = 404)

    if f"create_{model}" in self._introspection[paper.__class__.__name__]:
      theModel = self._introspection[paper.__class__.__name__][f"create_{model}"]["consumes"]
    else:
      theModel = getattr(self._models, model.capitalize())

    try:
      consume = theModel.from_dict(request.json)
    except TypeError as e:
      return ErrorMessage(message = e.args, code = 400)

    perm = await self._models.Permission.get(self._table, context = paper.type, name = f"create_{model}")
    token = AuthToken.get(request.headers)
    actor = await token.get_actor(self._table, self.config["JWT_SECRET"], self._models.User)
    # actor = await self._models.User.get(self._table, slug = "garito")
    if not await perm.allows(actor, paper):
      return ErrorMessage(message = "Unauthorized", code = 401)

    member = getattr(paper, f"create_{model}", None)
    if member is None:
      member = self._generic_factory
      args = [request, paper, actor, consume]
    else:
      known = {"request": request, "actor": actor, "consume": consume}
      args = [known[param] for param in signature(member).parameters if param in known.keys()]

    try:
      result = await member(*args)
      if isinstance(result, Tree):
        result = result.to_plain_dict()
      return OkResult(result = result, code = 201)
    except DuplicateKeyError:
      return ErrorMessage(message = f"{consume.name} already exists @ {paper.name}", code = 409)

  @timed
  async def remover(self, request: Request, path: str = None):
    path_ = f"/{path or ''}"
    try:
      paper = await self.get_path(path_, self._models, 0)
      paper._table = request.app._table
    except NotFound as e:
      return ErrorMessage(message = e.args[0], code = 404)

    perm = await self._models.Permission.get(self._table, context = paper.type, name = "remove")
    token = AuthToken.get(request.headers)
    actor = await token.get_actor(self._table, self.config["JWT_SECRET"], self._models.User)
    # actor = await self._models.User.get(self._table, slug = "garito")
    if not await perm.allows(actor, paper):
      return ErrorMessage(message = "Unauthorized", code = 401)

    member = getattr(paper, "remove")
    if member is None:
      member = self._generic_remover
      args = [request, paper, actor]
    else:
      args = [request, actor]

    try:
      result = await member(*args)
      return OkListResult(result = result)
    except Exception as e:
      message = format_exception(*exc_info()) if request.app.config.get("DEBUG", False) else str(e)
      for line in message:
        logger.error(line)
      return ErrorMessage(message = message, code = 500)

  async def _generic_factory(self, request: Request, paper: Mongo, actor, consume, update_roles: bool = True):
    # this could be executed in a mongo transaction
    await paper.create_child(consume, request.app._models)

    # This assumes the actor should be the owner
    # Here we could implement a mechanism to allow delegation (when a user makes and action on behalf another user)
    # Or perhaps is better to delegate this decision to the model
    roles = actor.roles
    roles.append(f"owner@{consume.get_url()}")
    if update_roles:
      await actor.update(request.app._models, roles = roles)

    return {"object": consume.to_plain_dict(), "actor_roles": roles}

  async def _generic_remover(self, request: Request, paper: Mongo, actor, update_roles: bool = True):
    # this assumes that the actor is the owner. Meh...
    roles = actor.roles
    roles.pop(roles.index(f"owner@{paper.get_url()}"))

    await paper.delete(request.app._models)
    await actor.update(request.app._models, roles = roles)

    return roles

  async def notify(self, request, notification: str, **kwargs) -> bool:
    member = getattr(self, notification, None)
    if member is None:
      raise NotFound(f"The server hasn't {notification} as notification")

    return await member(request, **kwargs)

  async def send_email(self, to: str, subject: str, from_: str = None, text: str = None, html: str = None, attachments: Any = None) -> bool:
    if from_ is None:
        from_ = self.config["MAIL_SENDER"]

    if self.config.get("DEBUG_NOTIFICATIONS", False):
      logger.info(to)
      logger.info(from_)
      logger.info(subject)
      logger.info(text)
      logger.info(html)
      logger.info(attachments)
    else:
      if text is None and html is None:
        raise ValidationError("Neither text nor html has been provided")
      elif text is not None and html is not None:
        message = MIMEMultipart("alternative")
        message.attach(MIMEText(text))
        message.attach(MIMEText(html, "html"))
      elif html is not None and attachments:
        message = MIMEMultipart()
        message.attach(MIMEText(html, "html"))
      elif text is None:
        message = MIMEText(html, "html")
      else:
        message = MIMEText(text)

      if attachments:
        for idx, attachment in enumerate(attachments):
          if isfile(attachment):
            m_type = guess_type(attachment)[0]
            main_type, subtype = m_type.split('/', 1)
            attach = MIMEBase(main_type, subtype)
            with open(attachment, 'rb') as f:
              attach.set_payload(f.read())
            encode_base64(attach)
            attach.add_header("Content-Disposition", "attachment", filename = attachment)
            attach.add_header("X-Attachment-Id", str(idx))
            attach.add_header("Content-ID", f"<{idx}>")
            message.attach(attach)
          else:
            logger.warn(f"{attachment} is not a file")

      message["From"] = from_
      message["To"] = to
      message["Subject"] = subject

      return await send(message, hostname = self.config["MAIL_SERVER"], port = self.config["MAIL_PORT"], **self.config.get("MAIL_ARGS", {}))

  async def _generic_options(self, request, *args, **kwargs):
    return response.text("", status = 204)

  async def _allow_origin(self, request, response):
    response.headers["Access-Control-Allow-Origin"] = "*"
    response.headers["Access-Control-Allow-Methods"] = "GET, POST, PUT, DELETE, OPTIONS"
    response.headers["Access-Control-Allow-Headers"] = "Access-Control-Allow-Origin, Access-Control-Allow-Headers, Origin, X-Requested-With, Content-Type, Authorization"

class MongoServer(ySanic):
  def __init__(self, root_model: Tree, models: ModuleType, **kwargs: Dict[str, Any]):
    super().__init__(root_model, models, **kwargs)

    self.register_listener(self._set_table, 'before_server_start')
    self.register_listener(self._close_table, 'before_server_stop')

  async def _set_table(self, app, loop):
    app._client = AsyncIOMotorClient(app.config["MONGO_URI"], io_loop = loop)
    db = app._client[app.config["MONGO_DB"]]
    app._table = db[app.config.get("MONGO_TABLE", app.config["MONGO_DB"])]

    if app.config.get("MONGO_GRIDFS", False):
      app._gridfs = AsyncIOMotorGridFSBucket(db)

    app._table.create_index("created_at", expireAfterSeconds = 1800)
    app._table.create_index([("path", ASCENDING), ("slug", ASCENDING)], unique = True)

    root = await app._root_model.get(app._table, path = "")
    if root:
      await root._rebuild_sec(app)

  def _close_table(self, app, loop):
    app._client.close()


import streamlit as st

from front.model_controller import get_event_push

def rest_alert(default_current_time):
    st.header("기사님 휴식 푸시 메세지")

    # st.image('resources/weather_alert_flow.png')

    current_time = st.text_input("시간 설정", default_current_time)

    # 버튼 클릭 여부 확인
    if st.button("모델 예측", key="rest_alert"):
        with st.spinner("Loading..."):
            recommend_result = get_event_push(event_type="장시간 근무 알람", current_time=current_time)
            st.write(recommend_result)


from mock import Mock, patch, ANY, sentinel
from nose.tools import ok_, eq_, raises

from operant.base import Registry

class RegObject(object):
    def __init__(self, name):
        self.name = name

class TestRegistry(object):

    @property
    def _reg(self):
        return Registry("testing", "name")

    @raises(RuntimeError)
    def test_register_existing(self):
        a = RegObject("foo")
        b = RegObject("foo")
        reg = self._reg
        reg.register(a)
        # This line should raise
        reg.register(b)

    def test_register(self):
        reg = self._reg
        a = RegObject("bar")
        reg.register(a)
        eq_(reg.get("bar"), a)

    def test_get_none(self):
        eq_(self._reg.get("__not__valid__"), None)

    def _handler_res(self):
        m = Mock()
        m.name = "handler_res"
        return m

    def test_str_handler(self):
        reg = self._reg

        res = self._handler_res()
        handler = Mock(return_value=res)

        reg.set_str_handler(handler)

        reg.register("whatever")
        handler.assert_called_once_with("whatever")
        eq_(reg.get("handler_res"), res)

    def test_cust_handler(self):
        reg = self._reg

        res = self._handler_res()
        handler = Mock(return_value=res)

        class Foo(object):
            name = "Foo"
        reg.set_handler(Foo, handler)

        foo = Foo()
        reg.register(foo)

        handler.assert_called_once_with(foo)
        eq_(reg.get("handler_res"), res)


from skimage.transform import rescale
import matplotlib.pyplot as plt

# Loading the image using Matplotlib
image = plt.imread('garden.jpg')

#Rescale the image to be 1/4
image_rescaled = rescale(image, 1/4, anti_aliasing=True, multichannel=True)

def show_image(image, title):
    plt.imshow(image)
    plt.title(title)
    plt.axis('off')
    plt.show()

show_image(image, 'orginal')
show_image(image_rescaled, 'Rescale image')


import os
import glob
import argparse
import numpy as np

class LabelToKittiConverter:
    """ KITTI FORMAT
    #Values    Name      Description
    ----------------------------------------------------------------------------
       1    type         Describes the type of object: 'Car', 'Van', 'Truck',
                         'Pedestrian', 'Person_sitting', 'Cyclist', 'Tram',
                         'Misc' or 'DontCare'
       1    truncated    Float from 0 (non-truncated) to 1 (truncated), where
                         truncated refers to the object leaving image boundaries
       1    occluded     Integer (0,1,2,3) indicating occlusion state:
                         0 = fully visible, 1 = partly occluded
                         2 = largely occluded, 3 = unknown
       1    alpha        Observation angle of object, ranging [-pi..pi]
       4    bbox         2D bounding box of object in the image (0-based index):
                         contains left, top, right, bottom pixel coordinates
       3    dimensions   3D object dimensions: height, width, length (in meters)
       3    location     3D object location x,y,z in camera coordinates (in meters)
       1    rotation_y   Rotation ry around Y-axis in camera coordinates [-pi..pi]
       1    score        Only for results: Float, indicating confidence in
                         detection, needed for p/r curves, higher is better.
    """
    def __init__(self, conversion, labels_fp, gt_labels_fp, csv):
        conversion_dict = self.get_attribute_idx(conversion)
        self.csv = csv
        if labels_fp is not None:
            print("------ Prediction Labels Conversion -------")
            new_labels_fp = (os.path.join(labels_fp, '..',  'pred_labels_kitti/data/'))
            if new_labels_fp[0] == '/':
                new_labels_fp = new_labels_fp[1:]
            os.makedirs(new_labels_fp, exist_ok=True)
            self.convert_to_kitti(conversion_dict, labels_fp, new_labels_fp, pred_labels=True)
        if gt_labels_fp is not None:
            print("------ Ground Truth Labels Conversion -------")
            new_labels_fp = os.path.join(gt_labels_fp, '..',  '/gt_labels_kitti/')
            if new_labels_fp[0] == '/':
                new_labels_fp = new_labels_fp[1:]
            os.makedirs(new_labels_fp, exist_ok=True)
            self.convert_to_kitti(conversion_dict, gt_labels_fp, new_labels_fp, pred_labels=False)

    def convert_to_kitti(self, conversion_key, fp, new_fp, pred_labels=False):
        labels_list = glob.glob(fp + "*")
        for num, example in enumerate(labels_list):
            print("{}/{}\r".format(num+1, len(labels_list)))
            example_idx = os.path.basename(example)
            new_label = self.new_label_from_txt(example, conversion_key, pred=pred_labels)
            if pred_labels:
                np.savetxt(new_fp + example_idx, new_label, delimiter=' ',
                           fmt='%s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s')
            else:
                np.savetxt(new_fp + example_idx, new_label, delimiter=' ',
                           fmt='%s %s %s %s %s %s %s %s %s %s %s %s %s %s %s')

    def new_label_from_txt(self, label_path, idx_key, pred=True):
        classes = []
        trunc = []
        occ = []
        obs = []
        camera_box = []
        score = []
        x, y, z, r = [], [], [], []
        l, w, h = [], [], []
        with open(label_path, "r") as f:
            labels = f.read().split("\n")
            for label in labels:
                if not label:
                    continue
                if self.csv:
                    label = label.replace(" ", "")
                    label = label.split(",")
                else:
                    label = label.replace(",", "")
                    label = label.split(" ")

                if 'class' in idx_key:
                    classes.append(label[idx_key['class']])
                else:
                    classes.append(['Car']) # assume if no class is specified, its a car
                if 'truncated' in idx_key:
                    trunc.append(label[idx_key['truncated']])
                else:
                    trunc.append(0.00)
                if 'occluded' in idx_key:
                    occ = occ.append(label[idx_key['occluded']])
                else:
                    occ.append(0)
                if 'alpha' in idx_key:
                    obs.append(label[idx_key['alpha']])
                else:
                    obs.append(0)
                if 'x1' in idx_key:
                    start, end = idx_key['x1'], idx_key['x1']+4
                    camera_box.append(label[start:end])
                else:
                    camera_box.append((0, 0, 50, 50))
                if 'x' in idx_key:
                    x.append(label[idx_key['x']])
                else:
                    x.append(0)
                if 'y' in idx_key:
                    y.append(label[idx_key['y']])
                else:
                    y.append(0)
                if 'z' in idx_key:
                    z.append(label[idx_key['z']])
                else:
                    z.append(0)
                if 'r' in idx_key:
                    r.append(label[idx_key['r']])
                else:
                    r.append(0)
                if 'l' in idx_key:
                    l.append(label[idx_key['l']])
                else:
                    l.append(0)
                if 'w' in idx_key:
                    w.append(label[idx_key['w']])
                else:
                    w.append(0)
                if 'h' in idx_key:
                    h.append(label[idx_key['h']])
                else:
                    h.append(0)
                if pred:
                    if 'score' in idx_key:
                        score.append(label[idx_key['score']])
        final_array = np.hstack((
            np.array(classes).reshape(-1, 1),
            np.array(trunc).reshape(-1, 1),
            np.array(occ).reshape(-1,1 ),
            np.array(obs).reshape(-1, 1),
            np.array(camera_box),
            np.array(h).reshape(-1, 1),
            np.array(w).reshape(-1, 1),
            np.array(l).reshape(-1, 1),
            np.array(x).reshape(-1, 1),
            np.array(y).reshape(-1, 1),
            np.array(z).reshape(-1, 1),
            np.array(r).reshape(-1, 1)
        ))
        if pred:
            final_array = np.hstack((final_array, np.array(score).reshape(-1, 1)))
        return final_array

    @staticmethod
    def get_attribute_idx(conversion):
        idx_dict = {}
        conversion = conversion.split(" ")
        print("-- Your conversion key --")
        for i, attribute in enumerate(conversion):
            print(i, attribute)
            if attribute == 'class':
                idx_dict['class'] = i
            elif attribute == 'truncated':
                idx_dict['truncated'] = i
            elif attribute == 'occluded':
                idx_dict['occluded'] = i
            elif attribute == 'alpha':
                idx_dict['alpha'] = i
            elif attribute == 'x1':
                idx_dict['x1'] = i # assumes [x1 y1 x2 y2] follows
            elif attribute == 'x':
                idx_dict['x'] = i
            elif attribute == 'y':
                idx_dict['y'] = i
            elif attribute == 'z':
                idx_dict['z'] = i
            elif attribute == 'l':
                idx_dict['l'] = i
            elif attribute == 'w':
                idx_dict['w'] = i
            elif attribute == 'h':
                idx_dict['h'] = i
            elif attribute == 'r':
                idx_dict['r'] = i
            elif attribute == 'score':
                idx_dict['score'] = i
        return idx_dict

def main():
    parser = argparse.ArgumentParser(description='arg parser')
    parser.add_argument('--pred_labels', type=str, help='file path with your prediction labels')
    parser.add_argument('--gt_labels', type=str, help='file path with your ground truth labels')
    parser.add_argument('--format', type=str, help='your label format e.g. "class, x, y, z, l, w, h"', required=True)
    parser.add_argument('--csv', dest='csv', action='store_true', help='is your file csv or space separated')
    args = parser.parse_args()
    print("------ Converting to KITTI Labels -------")
    LabelToKittiConverter(args.format, args.pred_labels, args.gt_labels, csv=args.csv)

if __name__ == '__main__':
    main()


from detectron2.data import DatasetCatalog, MetadataCatalog
from path import Path
from detectron2.data.datasets.coco import  load_coco_json

ROOT_DIR = Path(__file__).abspath().parent.parent.parent.parent

DATA_ROOT = ROOT_DIR / "datasets"

DATASETS_TO_BUILD = {
    "MOT17_train": (DATA_ROOT / "MOT17" / "train", DATA_ROOT / "MOT17" / "annotations" / "train_tmp.json"),
    "MOT17_val": (DATA_ROOT / "MOT17" / "val", DATA_ROOT / "MOT17" / "annotations" / "val_tmp.json"),
    "MOT20_train": (DATA_ROOT / "MOT20" / "train", DATA_ROOT / "MOT20" / "annotations" / "train.json"),
    "MOT20_val": (DATA_ROOT / "MOT20" / "val", DATA_ROOT / "MOT20" / "annotations" / "val.json"),
}

THING_CLASSES = ["person"]

for dataset_name, (image_root, json_file) in DATASETS_TO_BUILD.items():
    image_root = str(image_root)
    json_file = str(json_file)
    DatasetCatalog.register(
        dataset_name,
        lambda image_root=image_root, json_file=json_file: load_coco_json(json_file, image_root)
    )
    MetadataCatalog.get(dataset_name).set(
        thing_classes=THING_CLASSES, image_root=image_root, json_file=json_file
    )

if __name__ == "__main__":
    # print(ROOT_DIR)
    # load datasets
    datasets = DatasetCatalog.get("MOT17_train")
    meta = MetadataCatalog.get("MOT17_train")
    print(datasets[0])


import os

import pytest
import pandas as pd
from datetime import datetime, date
from rules import rule_price_earning_ratio

@pytest.mark.parametrize("date_range, from_date_str, outcome", [
    ("1y", "Oct 23 2021", "Oct 23 2020"),
    ("2 years", "Oct 23 2021", "Oct 24 2019"),
    ("1m", "Oct 23 2021", "Sep 23 2021"),
    ("2months", "Oct 24 2021", "Aug 25 2021"),
    ("1d", "Oct 23 2021", "Oct 22 2021"),
    ("2 days", "Oct 23 2021", "Oct 21 2021")
])
def test_extract_date(date_range, from_date_str, outcome):
    from_date = datetime.strptime(from_date_str, "%b %d %Y")
    outcome_date = datetime.strptime(outcome, "%b %d %Y")
    assert outcome_date == rule_price_earning_ratio.extract_date(date_range, from_date)

@pytest.fixture
def load_stock_a():
    stock_a = os.path.join(os.getcwd(), "rules/tests/", "test_data_stock_a.csv")
    df_stock_a = pd.read_csv(stock_a)
    df_stock_a['date'] = pd.to_datetime(df_stock_a['date']).dt.date
    return df_stock_a

@pytest.mark.parametrize("percentile_threshold, date_range, outcome", [
    ("0.1", "1m", False),
    ("0.2", "1m", True),
    ("0.5", "1m", True),
])
def test_filter_by_percentile(load_stock_a, percentile_threshold, date_range, outcome):
    assert outcome == rule_price_earning_ratio.filter_by_percentile(load_stock_a, percentile_threshold, date_range)


from contextlib import contextmanager
import inspect
import random as py_random
from typing import Iterable

import numpy as np

from timeserio.externals import tensorflow as tf, keras

def iterlayers(model: keras.layers.Layer) -> Iterable[keras.layers.Layer]:
    """
    Return iterable over all layers (and sub-layers) of a model.

    This works because a keras Model is a sub-class of Layer.
    Can be used for freezing/un-freezing layers, etc.
    """
    if hasattr(model, 'layers'):
        for layer in model.layers:
            yield from iterlayers(layer)
    else:
        yield model

def has_arg(fn, name, accept_all=False):
    """Check if a callable accepts a given keyword argument.

    See https://github.com/tensorflow/tensorflow/pull/37004

    Arguments:
      fn: Callable to inspect.
      name: Check if `fn` can be called with `name` as a keyword argument.
      accept_all: What to return if there is no parameter called `name` but the
        function accepts a `**kwargs` argument.
    Returns:
      bool, whether `fn` accepts a `name` keyword argument.
    """
    arg_spec = inspect.getfullargspec(fn)
    if accept_all and arg_spec.varkw is not None:
        return True
    return name in arg_spec.args or name in arg_spec.kwonlyargs

@contextmanager
def seed_random(seed=42):
    """Seed all random number generators to ensure repeatable tests.

    Sets python, `numpy`, and `tensorflow` random seeds
    to a repeatable states. This is useful in tests, but should not be
    used in production.
    """
    py_random.seed(seed)
    np.random.seed(seed)
    tf.random.set_seed(seed)
    yield


import asyncio, time

async def worker(delay, name):
    # эмитируем ожидание какого-то ответа
    # ответа от стороннего сервиса
    await asyncio.sleep(delay)
    # возвращаем результат
    return name, delay

async def main():
    # список аргументов для `worker()``
    targets = [(0.5, 'one'), (0.8, 'two'),
               (0.2, 'three'), (1, 'four'), (0.5, 'five')]

    # создаем задачи
    tasks = [asyncio.create_task(worker(*target)) for target in targets]

    # передаем задачи в функцию `as_completed()`
    for future in asyncio.as_completed(tasks, timeout=1.1):
        # получаем результаты по готовности
        res = await future
        # выводим на печать
        print(f'Задача: {res[0]}; задержка: {res[1]}')

if __name__ == '__main__':
    start = time.time()
    # запускаем цикл событий
    asyncio.run(main())

    total = time.time() - start
    print(f'Общее время выполнения: {total:.3f} сек.')

# Задача: three; задержка: 0.2
# Задача: one; задержка: 0.5
# Задача: five; задержка: 0.5
# Задача: two; задержка: 0.8
# Задача: four; задержка: 1
# Общее время выполнения: 1.001 сек.


import numpy as np
from torchvision import datasets, transforms
from .imagenetcls import ImageNetClsDataset
from PIL import Image

from io import BytesIO
import cv2

def disk(radius, alias_blur=0.1, dtype=np.float32):
    if radius <= 8:
        L = np.arange(-8, 8 + 1)
        ksize = (3, 3)
    else:
        L = np.arange(-radius, radius + 1)
        ksize = (5, 5)
    X, Y = np.meshgrid(L, L)
    aliased_disk = np.array((X ** 2 + Y ** 2) <= radius ** 2, dtype=dtype)
    aliased_disk /= np.sum(aliased_disk)

    # supersample disk to antialias
    return cv2.GaussianBlur(aliased_disk, ksize=ksize, sigmaX=alias_blur)

def shotnoise(x, severity=1):
    c = [500, 250, 100, 75, 50][severity - 1]

    x = np.array(x) / 255.
    return np.clip(np.random.poisson(x * c) / c, 0, 1) * 255

def defocusblur(x, severity=1):
    c = [(0.3, 0.4), (0.4, 0.5), (0.5, 0.6), (1, 0.2), (1.5, 0.1)][severity - 1]

    x = np.array(x) / 255.
    kernel = disk(radius=c[0], alias_blur=c[1])

    channels = []
    for d in range(3):
        channels.append(cv2.filter2D(x[:, :, d], -1, kernel))
    channels = np.array(channels).transpose((1, 2, 0))  # 3x32x32 -> 32x32x3

    return np.clip(channels, 0, 1) * 255

def jpeg(x, severity=1):
    c = [80, 65, 58, 50, 40][severity - 1]

    output = BytesIO()
    x.save(output, 'JPEG', quality=c)
    x = Image.open(output)

    return x

class ImageShuffler:
    # set seed=None to use default random state
    def __init__(self, mode, ny, nx, h=32, w=32, seed=42):
        assert h % ny == 0, 'h should be divisible by ny'
        assert w % nx == 0, 'w should be divisible by nx'
        self.mode = mode
        self.ny, self.nx = ny, nx
        self.h, self.w = h, w
        dy, dx = h // ny, w // nx
        p = np.empty([h, w], dtype=np.int32)
        r = np.arange(h * w).reshape(h, w)
        if mode == 'glo':
            q = np.random.RandomState(seed).permutation(ny * nx).reshape(ny, nx)
            for y in range(ny):
                for x in range(nx):
                    z = q[y, x]
                    v, u = z // nx, z % nx
                    p[y*dy:(y+1)*dy, x*dx:(x+1)*dx] = r[v*dy:(v+1)*dy, u*dx:(u+1)*dx]
        elif mode == 'loc':
            q = np.random.RandomState(seed).permutation(dy * dx)
            for y in range(ny):
                for x in range(nx):
                    v, u = y, x
                    p[y*dy:(y+1)*dy, x*dx:(x+1)*dx] = r[v*dy:(v+1)*dy, u*dx:(u+1)*dx].reshape(-1)[q].reshape(dy,dx)
        self.p = p.reshape(-1)
        self.p_inv = np.empty(h*w, dtype=np.int32)
        self.p_inv[self.p] = r.reshape(-1)

    # img: shape (h,w,c)
    def apply_one(self, img):
        s = img.shape
        assert len(s) == 3 and s[0] == self.h and s[1] == self.w
        return img.reshape(-1, s[-1])[self.p].reshape(s)

    # imgs: shape (n,h,w,c)
    def apply_batch(self, imgs):
        s = imgs.shape
        assert len(s) == 4
        if s[1] == self.h and s[2] == self.w:
            return imgs.reshape(s[0], -1, s[-1])[:, self.p].reshape(s)
        elif s[2] == self.h and s[3] == self.w:
            return imgs.reshape(s[0], s[1], -1)[:, :, self.p].reshape(s)

    def invert_one(self, img):
        s = img.shape
        return img.reshape(-1, s[-1])[self.p_inv].reshape(s)

    def invert_batch(self, imgs):
        s = imgs.shape
        return imgs.reshape(s[0], -1, s[-1])[:, self.p_inv].reshape(s)

def gamma_distortion_uint8(x, gamma):
    # print(x.dtype, x.shape, x.min(), x.max())  # uint8, (50000, 32, 32, 3) 0 255
    assert x.dtype == np.uint8
    return (255.*((x/255.)**gamma)).astype(np.uint8)

def corrupt_dataset(DatasetClass, corrupt):
    corrupt = corrupt.split('_')

    # if not corrupt or (corrupt.startswith('ori') and not resize):
    #     return DatasetClass
    # if not resize:
    if DatasetClass is datasets.CIFAR10 or DatasetClass is datasets.CIFAR100:
        image_size = 32

        class CorruptedDataset(DatasetClass):
            def __init__(self, *args, transform=None, **kwargs):
                super().__init__(*args, **kwargs)
                resize = transforms.Resize((image_size, image_size))
                # transform = transforms.Compose([transforms.ToTensor(),
                #                                 transforms.Normalize(mean=(0.4914, 0.4822, 0.4465), std=(0.247, 0.2435, 0.2616))])
                if corrupt[0] == 'glo' or corrupt[0] == 'loc':
                    shuf = ImageShuffler(corrupt[0], ny=int(corrupt[1]), nx=int(corrupt[1]), h=image_size, w=image_size)
                    def aug_func(x):
                        x = resize(x)
                        x = np.asarray(x)
                        x = shuf.apply_one(x)
                        x = Image.fromarray(x)
                        if transform is not None: x = transform(x)
                        return x
                elif corrupt[0] == 'randglo' or corrupt[0] == 'randloc':
                    def aug_func(x):
                        shuf = ImageShuffler(corrupt[0][-3:], ny=int(corrupt[1][-3:]), nx=int(corrupt[1][-3:]), h=image_size, w=image_size, seed=None)
                        x = resize(x)
                        x = np.asarray(x)
                        x = shuf.apply_one(x)
                        x = Image.fromarray(x)
                        if transform is not None: x = transform(x)
                        return x
                elif corrupt[0] == 'gam':
                    def aug_func(x):
                        x = resize(x)
                        x = np.asarray(x)
                        x = gamma_distortion_uint8(x, gamma=float(corrupt[1]))
                        x = Image.fromarray(x)
                        if transform is not None: x = transform(x)
                        return x
                elif corrupt[0] == 'ori':
                    def aug_func(x):
                        x = resize(x)
                        if transform is not None: x = transform(x)
                        return x
                else:
                    def aug_func(x):
                        x = resize(x)
                        x = eval(corrupt[0])(x, 3)
                        x = Image.fromarray(np.uint8(x))
                        if transform is not None: x = transform(x)
                        return x
                self.transform = aug_func

        return CorruptedDataset

    elif DatasetClass is datasets.STL10:
        image_size = 96
    elif DatasetClass is ImageNetClsDataset:
        image_size = 256

        class CorruptedDataset(DatasetClass):
            def __init__(self, *args, transform=None, **kwargs):
                super().__init__(*args, **kwargs)
                resize = transforms.Resize((image_size, image_size))
                if corrupt[0] == 'glo' or corrupt[0] == 'loc':
                    shuf = ImageShuffler(corrupt[0], ny=int(corrupt[1]), nx=int(corrupt[1]), h=image_size, w=image_size)
                    def aug_func(x):
                        x = resize(x)
                        x = np.asarray(x)
                        x = shuf.apply_one(x)
                        x = Image.fromarray(x)
                        if transform is not None: x = transform(x)
                        return x
                elif corrupt[0] == 'gam':
                    def aug_func(x):
                        x = resize(x)
                        x = np.asarray(x)
                        x = gamma_distortion_uint8(x, gamma=float(corrupt[1]))
                        x = Image.fromarray(x)
                        if transform is not None: x = transform(x)
                        return x
                else:
                    def aug_func(x):
                        x = resize(x)
                        x = eval(corrupt[0])(x, 3)
                        x = Image.fromarray(np.uint8(x))
                        if transform is not None: x = transform(x)
                        return x
                self.transform = aug_func

        return CorruptedDataset

    else:
        raise ValueError(f'unsupported {DatasetClass}')
    # else:
    #    image_size = resize

    class CorruptedDataset(DatasetClass):
        def __init__(self, *args, **kwargs):
            super().__init__(*args, **kwargs)
            # if resize:
            #     self.data = torch.nn.functional.interpolate(
            #         torch.as_tensor(self.data, dtype=torch.float32), [image_size, image_size], mode='bicubic').numpy()
            if corrupt[0] == 'glo' or corrupt[0] == 'loc':
                shuf = ImageShuffler(corrupt[0], ny=int(corrupt[1]), nx=int(corrupt[1]), h=image_size, w=image_size)
                self.data = shuf.apply_batch(self.data)
            elif corrupt[0] == 'gam':
                self.data = gamma_distortion_uint8(self.data, gamma=float(corrupt[1]))

    return CorruptedDataset


from django.urls import path, include

from profiller.api.view import ProfileListAPIView,ProfileStateListAPIView,ProfileIdListAPIView

urlpatterns = [
    path('', ProfileListAPIView.as_view(),name='list'),
    path('<int:pk>', ProfileIdListAPIView.as_view(),name='profile'),
    path('durum/<int:pk>', ProfileStateListAPIView.as_view(),name='profile-state'),
]


import boto3
from aws_cdk import (
    core as cdk,
    aws_events as events,
    aws_events_targets as events_targets,
    aws_iam as iam,
    aws_lambda as lmb,
    aws_logs as logs,
    aws_s3 as s3,
    aws_kinesisfirehose as kinesisfirehose,
    aws_securityhub as securityhub,
    custom_resources
)
from typing import Optional, List
from os import path

class SecurityHub(cdk.Construct):
    """Security Hub Contruct designed to act like an L2 CDK Construct"""

    def __init__(self, scope: cdk.Construct, identifier: str):
        super().__init__(scope, identifier)

        self.this_dir = path.dirname(__file__)
        enable_disable_function = lmb.Function(self, 'EnableSHFunction',
                                               code=lmb.Code.from_asset(path.join(self.this_dir,
                                                                                '../assets/lambdas/enable_security_hub_resource')),
                                               handler='index.handler',
                                               runtime=lmb.Runtime.PYTHON_3_8)

        enable_disable_function.add_to_role_policy(iam.PolicyStatement(
            effect=iam.Effect.ALLOW,
            actions=[
                'securityhub:EnableSecurityHub',
                'securityhub:DisableSecurityHub'
            ],
            resources=['*']
        ))

        enable_provider = custom_resources.Provider(self, 'EnableSHProvider',
                                                    on_event_handler=enable_disable_function,
                                                    log_retention=logs.RetentionDays.ONE_DAY)
        cdk.CustomResource(self, 'EnableSH',
                           service_token=enable_provider.service_token,
                           removal_policy=cdk.RemovalPolicy.RETAIN)
        self.__enabled = True

    @property
    def is_enabled(self):
        return self.__enabled

    def stream_raw_findings_to_s3(self,
                                  bucket_name: str,
                                  bucket_arn: str,
                                  bucket_region=None,
                                  raw_prefix='raw/firehose'):
        if bucket_region is None:
            bucket_region = cdk.Aws.REGION

        target_bucket = s3.Bucket.from_bucket_attributes(self, 'TargetBucket',
                                                         bucket_name=bucket_name,
                                                         bucket_arn=bucket_arn,
                                                         region=bucket_region
                                                         )

        role = iam.Role(self, 'DeliveryRole',
                        assumed_by=iam.ServicePrincipal('firehose.amazonaws.com'))

        target_bucket.grant_read_write(role)

        delivery_stream = kinesisfirehose.CfnDeliveryStream(self, 'SHDeliveryStream',
                                                            delivery_stream_type='DirectPut',
                                                            extended_s3_destination_configuration=kinesisfirehose.CfnDeliveryStream.ExtendedS3DestinationConfigurationProperty(
                                                                role_arn=role.role_arn,
                                                                bucket_arn=target_bucket.bucket_arn,
                                                                buffering_hints=kinesisfirehose.CfnDeliveryStream.BufferingHintsProperty(
                                                                    interval_in_seconds=900,
                                                                    size_in_m_bs=128
                                                                ),
                                                                compression_format='UNCOMPRESSED',
                                                                prefix=raw_prefix
                                                            ))

        stream_rule = events.Rule(self, 'StreamFromKinesisToS3',
                                  event_pattern=events.EventPattern(
                                      source=['aws.securityhub'],
                                      detail_type=['Security Hub Findings - Imported'],
                                  ))
        target = events_targets.KinesisFirehoseStream(
            stream=delivery_stream,
        )
        stream_rule.add_target(target)

    def enable_import_findings_for_product(self, product_arn):
        this_dir = path.dirname(__file__)

        enable_disable_function = lmb.Function(self, 'EnableSHImportFunction',
                                               code=lmb.Code.from_asset(path.join(self.this_dir,
                                                                                '../assets/lambdas/enable_import_prowler_findings')),
                                               handler='index.handler',
                                               runtime=lmb.Runtime.PYTHON_3_8)

        enable_disable_function.add_to_role_policy(iam.PolicyStatement(
            effect=iam.Effect.ALLOW,
            actions=['securityhub:EnableImportFindingsForProduct'],
            resources=['*']
        ))

        enable_provider = custom_resources.Provider(self, 'EnableSHImportProvider',
                                                    on_event_handler=enable_disable_function,
                                                    log_retention=logs.RetentionDays.ONE_DAY)

        cdk.CustomResource(self, 'EnableSHImport',
                           service_token=enable_provider.service_token,
                           properties={
                               'product_arn': product_arn
                           },
                           removal_policy=cdk.RemovalPolicy.RETAIN)

    # FIXME: If it is already created, then we need to check the region and possibly delete it.
    def enable_aggregation(self, regions: Optional[List[str]] = None):
        if regions is not None and len(regions) > 0:
            on_create = custom_resources.AwsSdkCall(
                action='createFindingAggregator',
                service='SecurityHub',
                parameters={
                    'RegionLinkingMode': 'SPECIFIED_REGIONS',
                    'Regions': regions
                },
                physical_resource_id=custom_resources.PhysicalResourceId.of(cdk.Aws.REGION)
            )

        else:
            on_create = custom_resources.AwsSdkCall(
                action='createFindingAggregator',
                service='SecurityHub',
                parameters={
                    'RegionLinkingMode': 'ALL_REGIONS',
                },
                physical_resource_id=custom_resources.PhysicalResourceId.of(cdk.Aws.REGION)
            )
        custom_resources.AwsCustomResource(self, 'SecurityHubAggregation',
                                           on_create=on_create,
                                           policy=custom_resources.AwsCustomResourcePolicy.from_sdk_calls(
                                               resources=custom_resources.AwsCustomResourcePolicy.ANY_RESOURCE
                                           ))


import datetime
from pymongo import MongoClient

# Connection with MongoClient
client = MongoClient('localhost', 27017)

# Getting a database (can be accessed with as property like client.my_database)
db = client['my_database']

# Getting a collection
collection = db.my_collection

# Documents
document = {
    "author": "yusa",
    "text": "My fist post",
    "tags": ["physics", "python", "deep learning"],
    "date": datetime.datetime.utcnow()
}
post_id = collection.insert_one(document).inserted_id
print(post_id)

# Find a document
post = collection.find_one({"author": "yusa"})
print(post)


# coding: utf-8

# In[9]:

import numpy as np
import pandas as pd
import random
from sklearn.cross_validation import train_test_split

class Data:

    def __init__(self):

        self.old_users=['C2002002', 'C2011020', 'C2016004', 'C2016005', 'C2012003', 'C2016003', 'C2012011', 'C1001001',
          'C2016014', 'C2018002', 'C2012001', 'C2012002', 'C2002007', 'C2003001', 'C2000007', 'C2016009',
          'C2028001', 'C2002010', 'C2017001', 'C2001002', 'C2019001', 'C2002008', 'C2011021',
          'C2012004', 'C1007002', 'C2018001', 'C2004003']

        self.data=self.__get_data()

        tmp=self.data[['user','item']].values
        self.users=set() #users=old_users+new_users
        self.items=set() #items=old_items+new_items(buy and quote)
        for i in range(len(tmp)):
            user=tmp[i][0]
            item=tmp[i][1]
            self.users.add(user)
            self.items.add(item)

        self.new_users=[x for x in self.users if x not in self.old_users]

        self.newdf,self.train,self.test=self.__split_data()

        self.dic,self.test_dic=self.__convert_dic() #train_dic,test_dic

        self.train_users=set()
        self.train_items=set()
        temp=self.train[['user','item']].values
        for i in range(len(temp)):
            user=temp[i][0]
            item=temp[i][1]
            self.train_users.add(user)
            self.train_items.add(item)

        self.user_industry=self.__user_industry() #user_industry={user:ind,user:ind,.....}
        self.item_pop=self.__item_pop()

    def __get_data(self):
        #在这里做了简单的打标签的特征工程，如果特征工程作的比较多，可以另外放一个类中进行调用即可
        df=pd.read_csv('E:\\Python2Projects_LOCAL\\yf_data\\yufang0508.csv')
        for col in ['item','industry']:
            df[col]=df[col].astype(str)
        df['item_len']=df['item'].apply(lambda x:int(len(x)))
        df['item_label_1']=0
        df['item_label_2']=0
        df.loc[df['item_len']==6,'item_label_1']=df['item'].apply(lambda x:str(x)[:1])
        df.loc[df['item_len']==7,'item_label_1']=df['item'].apply(lambda x:str(x)[:2])
        df.loc[df['item_len']==6,'item_label_2']=df['item'].apply(lambda x:str(x)[1:3])
        df.loc[df['item_len']==7,'item_label_2']=df['item'].apply(lambda x:str(x)[2:4])
        df['industry_str']='idx_'+df['industry']
        df=df.drop(['item_len'],axis=1)
        return df

    #根据最新调整的版本，如果是新用户，直接根据MostPopular进行推荐，然后在老用户中随机分成train+test二个部分
    def __split_data(self):
        new=[]
        old=[]
        tmp=self.data.values
        header=self.data.columns.tolist()
        for i in range(len(tmp)):
            if tmp[i][0] in self.new_users:
                new.append(tmp[i])
            else:
                old.append(tmp[i])
#         print("new_user samples len:",len(new))
#         print("old_user samples len:",len(old))
        new_data=pd.DataFrame(new,columns=header)
        old_data=pd.DataFrame(old,columns=header)

        seed=random.randint(0,2019)
        train,test=train_test_split(old_data,test_size=0.2,random_state=seed)
#         print("train samples len:",len(train))
#         print("test samples len:",len(test))

        return new_data,train,test #return:新用户df,traindf,testdf

    def __convert_dic(self):
        tmp_train=self.train[['user','is_quote','item']].values
        tmp_test=self.test[['user','is_quote','item']].values
        def get_dic(tmp):
            d={}
            for i in range(len(tmp)):
                user=tmp[i][0]
                is_quote=tmp[i][1]
                item=tmp[i][2]
                if user not in d:
                    d[user]={}
                if is_quote not in d[user]:
                    d[user][is_quote]=[]
                if item not in d[user][is_quote]:
                    d[user][is_quote].append(item)
            return d    #return:#{user:{0:[1,2,3],1:[3,4,5]}} ,0代表购买的item,1代表询价的item
        return get_dic(tmp_train),get_dic(tmp_test)

    def __user_industry(self): #由于user_ind是可以得到的，所以可以将所有用户放到一起得到dic
        user_industry={}
        tmp=self.data[['user','industry']].values
        for i in range(len(tmp)):
            user=tmp[i][0]
            ind=tmp[i][1]
            if user not in user_industry:
                user_industry[user]=0
            user_industry[user]=ind
        return user_industry

    def __item_pop(self):
        item_pop={}
        tmp=self.data[['is_quote','item']].values
        for i in range(len(tmp)):
            is_quote=tmp[i][0]
            item=tmp[i][1]
            if is_quote==0:
                if item not in item_pop:
                    item_pop[item]=0
                item_pop[item]+=1
            elif is_quote==1:
                if item not in item_pop:
                    item_pop[item]=0
                item_pop[item]+=0.5
        return item_pop


import tensorflow as tf

from tensorflow.keras import layers

import tensorflow_probability as tfp

tfd = tfp.distributions

# For numeric stability, set the default floating-point dtype to float64
tf.keras.backend.set_floatx('float64')
NUMBER_TYPE = tf.float64  # or tf.float32

STD_MIN_VALUE = 1e-13  # the minimal number that the diffusivity models can have

class ModelBuilder:
    """
    Constructs neural network models with specified topology.
    """
    DIFF_TYPES = ["diagonal", "triangular", "spd"]

    @staticmethod
    def define_forward_model(n_input_dimensions, n_output_dimensions, n_layers, n_dim_per_layer, name,
                             activation="tanh", dtype=tf.float64):

        inputs = layers.Input((n_input_dimensions,), dtype=dtype, name=name + '_inputs')
        network_x = inputs
        for i in range(n_layers):
            network_x = layers.Dense(n_dim_per_layer, activation=activation, dtype=dtype,
                                     name=name + "_hidden/dense_{}".format(i))(network_x)
        network_output = layers.Dense(n_output_dimensions, dtype=dtype,
                                      name=name + "_output_mean", activation=None)(network_x)

        network = tf.keras.Model(inputs=inputs, outputs=network_output,
                                 name=name + "_forward_model")
        return network

    @staticmethod
    def define_gaussian_process(n_input_dimensions, n_output_dimensions, n_layers, n_dim_per_layer, name,
                                diffusivity_type="diagonal", activation="tanh", dtype=tf.float64):

        def make_tri_matrix(z):
            # first, make all eigenvalues positive by changing the diagonal to positive values
            #print("in-make_tri_matrix",z.shape) # n(n + 1)/2
            z = tfp.math.fill_triangular(z)
            '''
            https://www.tensorflow.org/probability/api_docs/python/tfp/math/fill_triangular
            fill_triangular([1, 2, 3, 4, 5, 6])
            # ==> [[4, 0, 0],
            #      [6, 5, 0],
            #      [3, 2, 1]]
            '''
            #print("in-make_tri_matrix",z.shape) # (None, n, n)
            z2 = tf.linalg.diag(tf.linalg.diag_part(z)) # (None, n, n)
            """
            tf.linalg.diag_part --> see doc --> takes n*n shape and gives n shaped output
            tf.linalg.diag - takes n shapes in ---> n*n out with non diagonal terms zero

            """
            z = z - z2 + tf.abs(z2)  # this ensures the values on the diagonal are positive
            return z

        def make_spd_matrix(z):
            z = make_tri_matrix(z)
            return tf.linalg.matmul(z, tf.linalg.matrix_transpose(z))

        #def make_cov_matrix

        inputs = layers.Input((n_input_dimensions,), dtype=dtype, name=name + '_inputs')
        gp_x = inputs
        for i in range(n_layers):
            gp_x = layers.Dense(n_dim_per_layer,
                                activation=activation,
                                dtype=dtype,
                                name=name + "_mean_hidden_{}".format(i))(gp_x)
        gp_output_mean = layers.Dense(n_output_dimensions, dtype=dtype,
                                      name=name + "_output_mean", activation=None)(gp_x)

        # initialize with extremely small (not zero!) values so that it does not dominate the drift
        # estimation at the beginning of training
        small_init = 1e-2
        initializer = tf.keras.initializers.RandomUniform(minval=-small_init, maxval=small_init, seed=None)

        gp_x = inputs
        for i in range(n_layers):
            gp_x = layers.Dense(n_dim_per_layer,
                                activation=activation,
                                dtype=dtype,
                                kernel_initializer=initializer,
                                bias_initializer=initializer,
                                name=name + "_std_hidden_{}".format(i))(gp_x)
        if diffusivity_type=="diagonal":
            gp_output_std = layers.Dense(n_output_dimensions,
                                         kernel_initializer=initializer,
                                         bias_initializer=initializer,
                                         activation=lambda x: tf.nn.softplus(x) + STD_MIN_VALUE,
                                         name=name + "_output_std", dtype=dtype)(gp_x)
            #print("-diagonal-std-shape", gp_output_std.shape) # shape[none, n]
        elif diffusivity_type=="triangular":
            # the dimension of std should be N*(N+1)//2, for one of the Cholesky factors L of the covariance,
            # so that we can create the lower triangular matrix with positive eigenvalues on the diagonal.
            gp_output_tril = layers.Dense((n_output_dimensions * (n_output_dimensions + 1) // 2),
                                          activation="linear",
                                          kernel_initializer=initializer,
                                          bias_initializer=initializer,
                                          name=name + "_output_cholesky", dtype=dtype)(gp_x)
            gp_output_std = layers.Lambda(make_tri_matrix)(gp_output_tril)
            #print("-triangular-std-shape", gp_output_std.shape, gp_output_tril) # shape[none, n, n] shape[none, n(n+1)/2]
        elif diffusivity_type=="spd": # symmetric positive definite
            # the dimension of std should be N*(N+1)//2, for one of the Cholesky factors L of the covariance,
            # so that we can create the SPD matrix C using C = L @ L.T to be used later.
            gp_output_tril = layers.Dense((n_output_dimensions * (n_output_dimensions + 1) // 2),
                                          activation="linear",
                                          kernel_initializer=initializer,
                                          bias_initializer=initializer,
                                          name=name + "_output_spd", dtype=dtype)(gp_x)
            gp_output_std = layers.Lambda(make_spd_matrix)(gp_output_tril)
            print("-spd-std-shape", gp_output_std.shape, gp_output_tril)

        elif diffusivity_type=="spd_general": # symmetric positive definite
            # the dimension of std should be N*(N+1)//2, for one of the Cholesky factors L of the covariance,
            # so that we can create the SPD matrix C using C = L @ L.T to be used later.
            gp_output_tril = layers.Dense((n_output_dimensions * (n_output_dimensions + 1) // 2),
                                          activation="linear",
                                          kernel_initializer=initializer,
                                          bias_initializer=initializer,
                                          name=name + "_output_spd", dtype=dtype)(gp_x)
            gp_output_std = layers.Lambda(make_spd_matrix)(gp_output_tril)
            print("-spd-std-shape", gp_output_std.shape, gp_output_tril)
            # gp_output_std = layers.Lambda(lambda L: tf.linalg.matmul(L, tf.transpose(L)))(gp_output_tril)
        else:
            raise ValueError(f"Diffusivity type {diffusivity_type} not supported. Use one of {ModelBuilder.DIFF_TYPES}.")

        gp = tf.keras.Model(inputs,
                            [gp_output_mean, gp_output_std],
                            name=name + "_gaussian_process")
        return gp


#!/usr/bin/env python
# coding: utf-8

# # SECOND ASSIGNMENT
#
# **GROUP: EARTHQUAKES**
#
# Guadagnini Michele - 1230663
#
# Lambertini Alessandro - 1242885
#
# Pagano Alice - 1236916
#
# Puppin Michele - 1227474

# Import main packages
import pandas as pd
import numpy as np
import scipy as sp
import seaborn as sns
import matplotlib.pyplot as plt
import bokeh.palettes as palette # For palette of colors

import time
import os
import glob
from joblib import Parallel, delayed
import dask.dataframe as dd

class plot_hist:

        def mass(self,df,fMT,metal,ce_str,m_str): #(dataframe,fMT,metal)

                fig = plt.figure(figsize=(30,30))
                ax = [fig.add_subplot(3,3,s) for s in range(1,len(fMT)+1)] #nrows,ncols,index
                fig.suptitle('Plots of distribution of mass '+m_str+' '+ce_str, fontsize=30)
                col = sns.color_palette(palette.viridis(len(metal)))

                for i in range(0,len(fMT)):
                        for j in range(0,len(metal)):
                                #edge = np.logspace(np.log10(df[i][j]['mt1[4]'].min()), np.log10(df[i][j]['mt1[4]'].max()), 30)
                                sns.distplot(df[i][j][m_str],kde=False,norm_hist=False,label=metal[j],
                                                        hist_kws={"histtype": "step","lw": 2},ax=ax[i],color=col[j])

                        ax[i].set_title('Distribution of mass '+m_str+' for fMT={}'.format(fMT[i]),fontsize=18)
                        ax[i].set_xlabel('m $[M_\odot]$',fontsize=18)
                        ax[i].set_ylabel('Counts',fontsize=18)
                        #ax[i].set_xscale('log')
                        ax[i].legend(title='Metallicity',fontsize=14,title_fontsize=16,loc='upper right')
                        ax[i].tick_params(axis="x", labelsize=16)
                        ax[i].tick_params(axis="y", labelsize=16)
                        ax[i].grid(True, which="both", ls="-",color='0.93')
                        ax[i].set_axisbelow(True)

                if not os.path.isdir('./hist_fMT'):
                        os.mkdir('./hist_fMT')

                plt.savefig('hist_fMT/hist_'+m_str+'_fMT_'+ce_str+'.pdf', format='pdf')
                plt.close()

        def dtimes(self,df,fMT,metal,ce_str): #(dataframe,fMT,metal)

                fig = plt.figure(figsize=(30,30))
                ax = [fig.add_subplot(3,3,s) for s in range(1,len(fMT)+1)] #nrows,ncols,index
                fig.suptitle('Plots of distribution of delay times ' + ce_str,fontsize=30)
                col = sns.color_palette(palette.viridis(len(metal)))

                for i in range(0,len(fMT)):
                        for j in range(0,len(metal)):
                                #edge = np.logspace(np.log10(df[i][j]['t_step[1]'].min()), np.log10(df[i][j]['t_step[1]'].max()), 30)
                                sns.distplot(df[i][j]['dtimes'],kde=False,norm_hist=False,label=metal[j],
                                                        hist_kws={"histtype": "step","lw": 2},ax=ax[i],color=col[j])

                        ax[i].set_title('Distribution of delay times for fMT={}'.format(fMT[i]),fontsize=18)
                        ax[i].set_xlabel('t $[Myr]$',fontsize=18)
                        ax[i].set_ylabel('Counts',fontsize=18)
                        #ax[i].set_xscale('log')
                        #ax[i].set_yscale('log')
                        ax[i].legend(title='Metallicity',fontsize=14,title_fontsize=16,loc='upper right')
                        ax[i].tick_params(axis="x", labelsize=16)
                        ax[i].tick_params(axis="y", labelsize=16)
                        ax[i].grid(True, which="both", ls="-",color='0.93')
                        ax[i].set_axisbelow(True)

                if not os.path.isdir('./hist_fMT'):
                        os.mkdir('./hist_fMT')

                plt.savefig('hist_fMT/hist_dtimes_fMT_'+ce_str+'.pdf', format='pdf')
                plt.close()

        def frac_CE(self,df_CE,df_nCE,fMT,metal): #(dataframe,fMT,metal)

                num_fMT_CE = []
                num_fMT_nCE = []

                for i in range(len(fMT)):
                        num_CE = 0
                        num_nCE = 0
                        for j in range(len(metal)):
                                num_CE += df_CE[i][j].shape[0]
                                num_nCE += df_nCE[i][j].shape[0]
                        num_fMT_CE.append(num_CE)
                        num_fMT_nCE.append(num_nCE)

                frac_fMT_CE = [a/(a+b) for a,b in zip(num_fMT_CE,num_fMT_nCE)]
                frac_fMT_nCE = [b/(a+b) for a,b in zip(num_fMT_CE,num_fMT_nCE)]

                fMT = [0.1,0.2,0.3,0.4,0.5,0.7,1]
                fig = plt.figure(figsize=(10,10))
                ax = fig.add_subplot(111)
                fig.suptitle('Plots of fraction of CE BH system',fontsize=30)

                ax.plot(fMT,frac_fMT_CE,'.',color='#440154')
                ax.set_xticks([i/10 for i in range (1,11)])
                ax.set_title('Fraction of CE BH system')
                ax.set_xlabel('fMT')
                ax.set_ylabel('Fraction')
                ax.grid(color='0.93')

                if not os.path.isdir('./hist_fMT'):
                        os.mkdir('./hist_fMT')

                plt.savefig('hist_fMT/frac_CE.pdf', format='pdf')
                plt.close()


import os
import random
from sklearn.model_selection import train_test_split
import pickle
from viz_utils.utils import set_seed

set_seed(1234)

list_to_avoid = ["grafo6353.39.graphml", "grafo7223.40.graphml", "grafo7312.41.graphml"]
path_to_rome = os.path.join("data", "rome")

graph_files = os.listdir(path_to_rome)

graph_files = random.sample(graph_files, 100)

print("not_connected.txt" in graph_files)

for i in graph_files:
    if ".graphml" not in i:
        print(f"Removing {i}")
        graph_files.remove(i)

    if i in list_to_avoid:
        print(list_to_avoid)
        print(f"Removing {i}")
        graph_files.remove(i)

try:
    graph_files.remove("not_connected.txt")
    graph_files.remove("training_list")
    graph_files.remove("test_list")
    graph_files.remove("validation_list")
    graph_files.remove("training_list100")
    graph_files.remove("test_list100")
    graph_files.remove("validation_list100")
    graph_files.remove("training_list1000")
    graph_files.remove("test_list1000")
    graph_files.remove("validation_list1000")
    print("Conn removed!")
except:
    print("cannot find Not connected ")

print(len(graph_files))
print("not_connected.txt" in graph_files)

# training_set = random.sample(graph_files, 10000)
# remaining_set = [x for x in graph_files if x not in training_set]
#
# test_set = random.sample(remaining_set, 1000)
# validation = [x for x in remaining_set if x not in test_set]
#
# print(len(training_set))
# print(len(test_set))
# print(len(validation))
#
# print("Check if there are elem in common")
#
# common_tr_test = list(set(training_set).intersection(test_set))
# common_tr_val = list(set(training_set).intersection(validation))
# common_valr_test = list(set(validation).intersection(test_set))

train_ratio = 0.75
validation_ratio = 0.15
test_ratio = 0.10

X_train, X_test = train_test_split(graph_files, test_size=1 - train_ratio)
X_test, X_validate = train_test_split(X_test, test_size=test_ratio / (test_ratio + validation_ratio))

print(
    f"Len train: {len(X_train)}, len test: {len(X_test)}, len valid:{len(X_validate)}; total length: {len(X_train) + len(X_validate) + len(X_test)}")

with open(os.path.join(path_to_rome, "training_list100"), 'wb') as fp:
    pickle.dump(X_train, fp)
with open(os.path.join(path_to_rome, "validation_list100"), 'wb') as fp:
    pickle.dump(X_validate, fp)
with open(os.path.join(path_to_rome, "test_list100"), 'wb') as fp:
    pickle.dump(X_test, fp)


import os
from setuptools import setup

# Utility function to read the README file.
# Used for the long_description.  It's nice, because now 1) we have a top level
# README file and 2) it's easier to type in the README file than to put a raw
# string in below ...
def read(fname):
    return open(os.path.join(os.path.dirname(__file__), fname)).read()

setup(
    name = "firethorn",
    version = "0.1.0",
    author = "Stelios Voutsinas",
    author_email = "stv@roe.ac.uk",
    description = ("A python suite for firethorn, and vo services (TAP)"),
    license = "BSD",
    keywords = "firethorn vo",
    url = "http://wfau.metagrid.co.uk/code/firethorn",
    #packages=['firethorn'],
    include_package_data = True,
    packages=['firethorn.config', 'firethorn.utils', 'firethorn.models','firethorn.models.base','firethorn.models.jdbc','firethorn.models.ivoa','firethorn.models.adql','firethorn.models.adql_query', 'firethorn.tap', 'firethorn.core', 'firethorn'],
    long_description=read('README'),
    classifiers=[
        "Development Status :: 1 - Alpha",
        "Topic :: Utilities",
        "License :: GPL License",
    ],
    install_requires=[
        'numpy>=1.4.0',
        'astropy>=0.4.1',
        'requests'
    ]
)


import configparser
import os
from pyfiglet import Figlet
from sltools import hashing

'''
shortlink
sltools/setup.py
'''

version = "1.0.0"
custom_fig = Figlet(font="slant")
config = configparser.ConfigParser()

def start():
    print(custom_fig.renderText("shortlink"))
    print("Version: {0} \nLicense: MIT \nAuthor: berrysauce\n".format(version))
    try:
        with open("sl/config.ini", "r") as f:
            f.read()
    except IOError:
        print("[...]   Starting setup - no config found")
        print(50 * "-")
        print("Welcome to shortlink! Please follow this setup \nbefore you continue using shortlink.")
        print("You can change these settings later by editing the values in sl/config.ini.")

        print(" \n[1/7] What's your domain name? Press enter to use the default (localhost).")
        domain = input("(example.com) >>> ")
        if not domain:
            domain = "127.0.0.1"

        print(" \n[2/7] What's your desired port? Press enter to use the default (8000).")
        port = input("(number) >>> ")
        if not port:
            port = "8000"

        print(
            " \n[3/7] Where should we redirect users when they visit the root? You can use your homepage domain here. Press enter to use the default (/404).")
        redirect = input("(domain) >>> ")
        if not redirect:
            redirect = "/404"

        print(" \n[4/7] What logging level do you wish? Press enter to use the default (info).")
        logging = input("(info/debug) >>> ")
        if not logging:
            logging = "info"

        print(" \nexample.com/423dn3")
        print("   (This part ^^)")
        print(" \n[5/7] How long should the slashtag be? Press enter to use the default (7).")
        length = input("(number) >>> ")
        if not length:
            length = 7

        print(" \n[6/7] Should all links be publicly visible? Press enter to use the default (private).")
        visibility = input("(public/private) >>> ")
        if not visibility:
            visibility = "private"

        print(" \n[7/7] Please enter your Password for accessing the API. Keep this secret!")
        key = input("(your password) >>> ")

        print(50 * "-")
        print("[...]   Creating config")
        os.mkdir("sl")

        hashedkey = hashing.hashpw(key)

        config["SERVER"] = {"host": domain,
                            "port": port,
                            "redirect": redirect,
                            "length": length,
                            "logging": logging,
                            "visibility": visibility,
                            "key": hashedkey}
        with open("sl/config.ini", "w") as configfile:
            config.write(configfile)
        print("[...]   Created config")

        print("[...]   Creating database")
        with open("sl/db.json", "w") as database:
            database.write("")

        print("[...]   Created database")

    finally:
        print("[...]   Starting your server")


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import uuid
import argparse
import logging
import configparser

import xml.etree.ElementTree as ET
from xml.dom import minidom

from enum import Enum

import codecs
import urllib.parse

# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# Cue
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
class Cue:
    class Type(Enum):
        """
        Most cue types can be inferred:
          MemCue  -> num < 0
          Cue     -> num >=0
          + Loop  -> len > 0.0
        'Type' will be used to annotate extra info when needed.

        Values of Cue, FadeIn, FadeOut and Load identical in RB and TK.
        No Grid cue is available in RB.
        """
        Cue = 0
        FadeIn = 1
        FadeOut = 2
        Load = 3
        Grid = 5

    def __init__(self):
        self.name = ""
        self.start = 0.0
        self.len = 0.0
        self.num = -1
        self.type = Cue.Type.Cue

    def __str__(self):
        return "{} @{} [{}]".format(repr(self.type), self.start, self.num)

# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# Track
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
class Track:
    def __init__(self):
        self.id = ""
        self.name = ""
        self.artist = ""
        self.album = ""
        self.bpm = 120.0
        self.tonality = ""
        self.fileurl = ""
        self.cues = []

    def __str__(self):
        return "{}:\t{} ({}) [{}, {}, {} cues]" \
            .format(self.id, self.name, self.artist,
                    self.tonality, self.bpm, len(self.cues))

# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# Playlist
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
class Playlist:
    class Type(Enum):
        Folder = 0
        List = 1

    def __init__(self):
        self.name = ""
        self.type = Playlist.Type.Folder
        self.children = []

    def __str__(self):
        return self.__str_recursive(0)

    def __str_recursive(self, level):
        s = ""
        if self.type == Playlist.Type.Folder:
            s = "{}[{}]".format(' '*2*level, self.name)
            for c in self.children:
                s = s + "\n" + c.__str_recursive(level + 1)
        elif self.type == Playlist.Type.List:
            s = "{}{} ({} tracks)".format(' '*2*level, self.name, len(self.children))
            # for c in self.children:
            #     s = s + "\n{}- {}".format(' '*2*level, c)
        return s

# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# Library
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
class Library:
    def __init__(self):
        self.track_dict = {}
        self.playl_tree = None

# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# RekordboxReader
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
class RekordboxReader:
    def __init__(self):
        pass

    def read(self, path_xml : str) -> Library:
        l = Library()
        if not os.path.exists(path_xml):
            logging.error("File doesn't exist: {}".format(path_xml))
        else:
            logging.debug("Reading: {}".format(path_xml))
            l.track_dict = self.__parse_tracks(path_xml)
            l.playl_tree = self.__parse_playlists(path_xml)
        return l

    def __parse_tracks(self, path_xml):
        tree = ET.parse(path_xml)
        root = tree.getroot()
        tracks = {}

        coll_elem = root.find('COLLECTION')
        for track_elem in coll_elem.iter('TRACK'):
            a = track_elem.attrib
            t = Track()
            t.id = a['TrackID']
            t.name = a['Name']
            t.artist = a['Artist']
            t.bpm = float(a['AverageBpm'])
            t.tonality = a['Tonality']
            t.fileurl = a['Location']
            t.album = a['Album']

            for mark_elem in track_elem.iter('POSITION_MARK'):
                t.cues.append(self.__make_cue(mark_elem.attrib))

            t.cues.sort(key=lambda c: c.start)
            tracks[t.id] = t

        logging.debug("{} tracks found.".format(len(tracks)))
        return tracks

    def __make_cue(self, cue_dict) -> Cue:
        c = Cue()
        c.start = float(cue_dict['Start'])
        c.len = float(cue_dict['End']) - c.start if 'End' in cue_dict else 0.0
        c.num = int(cue_dict['Num'])
        c.name = cue_dict['Name']
        c.type = Cue.Type.Cue
        return c

    def __parse_playlists(self, path_xml) -> Playlist:
        tree = ET.parse(path_xml)
        root = tree.getroot()
        playl_root = None

        playl_elem = root.find('PLAYLISTS')
        for child in playl_elem:
            playl_root = self.__make_node_recursive(child)

        return playl_root

    def __make_node_recursive(self, node_elem) -> Playlist:
        a = node_elem.attrib
        p = Playlist()
        p.name = a['Name']
        p.type = Playlist.Type.Folder if a['Type'] == "0" else Playlist.Type.List

        for t_e in node_elem: # iterate over children
            if p.type == Playlist.Type.List and t_e.tag == "TRACK":
                p.children.append(t_e.attrib['Key'])
            elif p.type == Playlist.Type.Folder and t_e.tag == "NODE":
                p.children.append(self.__make_node_recursive(t_e))

        return p

# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# TraktorWriter
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
class TraktorWriter:
    def __init__(self):
        self.__sep = "/:"
        pass

    def write(self, lib : Library, path_xml : str):
        root = self.__init_dom()
        root = self.__render_tracks(root, lib)
        root = self.__render_playlists(root, lib)
        self.__write_to_output(path_xml, root)

    def __init_dom(self):
        root = ET.Element("NML", {"VERSION": "19"})
        for e in ["MUSICNODES", "COLLECTION", "PLAYLISTS", "SETS"]:
            ET.SubElement(root, e)
        return root

    def __generate_location(self, fileurl : str) -> dict:
        """
        Generates attribute dictionary for a LOCATION element from a file URL.
        {"DIR": ..., "FILE", ..., "VOLUME": ...}
        """
        url = urllib.parse.urlparse(fileurl)
        path = os.path.normpath(url.path)
        path = urllib.parse.unquote_plus(path)
        tokens = path.split(os.sep)

        locdict = {}
        locdict["VOLUME"] = tokens.pop(1)
        locdict["FILE"] = tokens.pop(-1)
        locdict["DIR"] = self.__sep.join(tokens)
        return locdict

    def __generate_cue(self, cue : Cue) -> dict:
        """
        Generates attribute dictionary for a CUE_V2 element from a file URL.
        {"NAME": ..., "TYPE", ..., "START": ...}
        """
        is_loop = cue.len > 0.0
        is_hot = cue.num > -1

        cuedict = {}
        cuedict["NAME"] = cue.name if cue.name != "" else \
                            "Loop" if is_loop else \
                            "Cue"  if is_hot else "Mem"
        cuedict["DISPL_ORDER"] = "0"
        cuedict["TYPE"] = str("5" if is_loop else cue.type.value)
        cuedict["START"] = str(cue.start*1000.0)
        cuedict["LEN"] = str(cue.len*1000.0)
        cuedict["REPEATS"] = "-1"
        cuedict["HOTCUE"] = str(cue.num)
        return cuedict

    def __generate_info(self, track : Track) -> dict:
        infodict = {}
        infodict["BITRATE"] = "320" # TODO
        infodict["KEY"] = track.tonality
        return infodict

    def __render_tracks(self, root, lib : Library):
        tracks = lib.track_dict.values()
        coll_elem = root.find('COLLECTION')
        coll_elem.attrib["ENTRIES"] = str(len(tracks))

        for t in tracks:
            t_e = ET.SubElement(coll_elem, "ENTRY")
            t_e.attrib['TITLE'] = t.name
            t_e.attrib['ARTIST'] = t.artist

            ET.SubElement(t_e, "LOCATION", self.__generate_location(t.fileurl))
            ET.SubElement(t_e, "ALBUM", {"TITLE": t.album})
            ET.SubElement(t_e, "INFO", self.__generate_info(t))
            ET.SubElement(t_e, "MODIFICATION_INFO", {"AUTHOR_TYPE": "user"})
            ET.SubElement(t_e, "TEMPO", {"BPM_QUALITY": "100",
                                         "BPM": str(t.bpm)})
            for c in t.cues:
                ET.SubElement(t_e, "CUE_V2", self.__generate_cue(c))

        return root

    def __generate_playl_track(self, track_id : str, track_dict : dict) -> dict:
        """
        Generates attribute dictionary for a PRIMARYKEY ENTRY of a PLAYLIST NODE.
        """
        track = track_dict[track_id]
        locdict = self.__generate_location(track.fileurl)
        attrib = {}
        attrib["KEY"] = locdict["VOLUME"] + locdict["DIR"] + self.__sep + locdict["FILE"]
        attrib["TYPE"] = "TRACK"
        return attrib

    def __generate_node_recursive(self, parent, playl : Playlist, track_dict : dict):
        """
        @param parent       Parent DOM node.
        @param playl        Current Playlist node.
        @param track_dict   Track dictionary to render track info.
        @return Modified parent DOM
        """
        node = ET.SubElement(parent, "NODE")
        node.attrib["NAME"] = "$ROOT" if playl.name == "ROOT" else playl.name

        if playl.type == Playlist.Type.Folder:
            node.attrib["TYPE"] = "FOLDER"
            subnode = ET.SubElement(node, "SUBNODES", {"COUNT": str(len(playl.children))})
            for c in playl.children:
                self.__generate_node_recursive(subnode, c, track_dict)
        elif playl.type == Playlist.Type.List:
            node.attrib["TYPE"] = "PLAYLIST"
            playlist = ET.SubElement(node, "PLAYLIST",
                                     {"ENTRIES": str(len(playl.children)),
                                      "TYPE": "LIST",
                                      "UUID": "/db/Playlist/" + str(uuid.uuid4())})
            for c in playl.children:
                entry = ET.SubElement(playlist, "ENTRY")
                ET.SubElement(entry, "PRIMARYKEY", self.__generate_playl_track(c, track_dict))

        return node

    def __render_playlists(self, root, lib : Library):
        if lib.playl_tree is not None:
            playl_elem = root.find('PLAYLISTS')
            self.__generate_node_recursive(playl_elem, lib.playl_tree, lib.track_dict)
        return root

    def __write_to_output(self, xml_path, root):
        xmlstr = minidom.parseString(ET.tostring(root)).toprettyxml(indent="  ")
        with codecs.open(xml_path, "w", "utf-8") as f:
            if f.write(xmlstr) is None:
                logging.info("Wrote output to file: {}".format(xml_path))
            else:
                logging.error("Failed to write to location: {}".format(xml_path))

# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
#
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #

class OptionalOperations:
    def __init__(self, config):
        self.config = config
        pass

    def apply(self, lib : Library) -> Library:
        lib.track_dict = self.__tk_add_grid_marker(lib.track_dict)
        lib.track_dict = self.__tk_assign_all_cues(lib.track_dict)
        return lib

    def __tk_add_grid_marker(self, tracks : dict) -> dict:
        if self.config.getboolean("Options", "GridMakerFromCue", fallback=False):
            for tid in tracks:
                t = tracks[tid]
                if len(t.cues) > 0:
                    gridcue = Cue()
                    gridcue.start = t.cues[0].start
                    gridcue.type = Cue.Type.Grid
                    gridcue.name = "Grid"
                    t.cues.insert(0, gridcue)
                    tracks[tid] = t
        return tracks

    def __tk_assign_all_cues(self, tracks : dict) -> dict:
        num = self.config.getint("Options", "AssignMemCueToPad", fallback=-1)
        if num > -1:
            for tid in tracks:
                t = tracks[tid]
                for i in range(0, len(t.cues)):
                    if t.cues[i].num < 0 and t.cues[i].type != Cue.Type.Grid:
                        t.cues[i].num = num
        return tracks

# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# main
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
if __name__ == "__main__":

    ''' Args '''
    parser = argparse.ArgumentParser(
                    prog='rb2tk.py',
                    description='Convert a rekordbox XML library into a '
                                'Traktor Pro 3 NML collection.',
                    epilog='For more info: github.com/martinbloedorn')

    parser.add_argument('RekordboxXmlInput', nargs='?', default=None,
                        help="Path to rekordbox XML library export.")
    parser.add_argument('TraktorNmlOutput', nargs='?', default=None,
                        help="Output path of generated Traktor NML collection.")
    parser.add_argument('-c', '--conf', action='store', default="rb2tk.ini")
    parser.add_argument('-v', '--verbose', action='count', default=0)

    args = parser.parse_args()

    ''' Logging '''
    logging.basicConfig(format='%(levelname)s @ %(funcName)s: %(message)s')
    levels = [logging.WARNING, logging.INFO, logging.DEBUG]
    logging.getLogger().setLevel(levels[min(args.verbose, len(levels)-1)])

    ''' Config '''
    config = configparser.ConfigParser()
    config.read(args.conf)

    if args.RekordboxXmlInput is not None:
        config["Library"]["RekordboxXmlInput"] = args.RekordboxXmlInput
    elif not config.has_option("Library", "RekordboxXmlInput"):
        config["Library"]["RekordboxXmlInput"] = "rekordbox.xml"

    if args.TraktorNmlOutput is not None:
        config["Library"]["TraktorNmlOutput"] = args.TraktorNmlOutput
    elif not config.has_option("Library", "TraktorNmlOutput"):
        config["Library"]["TraktorNmlOutput"] = "collection.nml"

    ''' Main '''
    rr = RekordboxReader()
    tw = TraktorWriter()
    oo = OptionalOperations(config)

    lib = rr.read(config["Library"]["RekordboxXmlInput"])
    lib = oo.apply(lib)
    tw.write(lib, config["Library"]["TraktorNmlOutput"])


#coding:utf-8
import sys
import time
import urllib
import urllib.request
import requests
from bs4  import BeautifulSoup
import os

# If we need 500 files, then we need 50 pages in page map
# mymap=['0','1','2','3','4','5','6','7']
# Take only one page to prevent to be put into black list
# page map
mymap=['0']

# access search result page
def baidu_search(key_words,pagenum):
    url='http://www.baidu.com/s?wd='+key_words+'&pn='+mymap[pagenum]
    html=urllib.request.urlopen(url).read()
    return html

# Send 'get' request to w3c validator
def validate(url):
    validaror_url = "https://validator.w3.org/nu/"
    params = { "doc": url, "out": "json"}
    response = requests.get(validaror_url,params=params)
    # Use "type:error" to locate errors
    error_key = "\"type\":\"error\""
    # Use "subType:warning" to locate warnings
    warning_key = "\"subType\":\"warning\""
    # .text contains entire content of checking result
    return str(response.text.count(error_key)), str(response.text.count(warning_key))

# parse html of result page
def deal_key(key_words):
    # create output file
    if os.path.exists('data')==False:
        os.mkdir('data')
    filename='data\\'+key_words+'.txt'
    fp=open(filename,'wb')
    if fp:
        pass
    else:
        print('Failed to create output file：'+filename)
        return

    # parse html of result page one by one
    x=0
    while x<=mymap.__len__()-1:
        htmlpage=baidu_search(key_words,x)
        soup=BeautifulSoup(htmlpage, "html.parser")
        # Find result links
        for item in soup.findAll("div", {"class": "result"}):
            a_click = item.find('a')
            if a_click:
                fp.write(a_click.get("href").encode('utf-8'))
                # collect errors and warnings
                err, war = validate(a_click.get("href").encode('utf-8'))
                fp.write(b' errors: %b, warnings: %b' % (bytes(err, 'utf8'),bytes(war, 'utf8')))
            fp.write(b'\r\n')
        x=x+1
        fp.write(b'\r\n')
    fp.close()

# entrance
print('Start:')
# search key word
Keyword = input("Please input keyword for searching: ")
deal_key(Keyword)
print('End！')


from django.conf import settings
from django.conf.urls import url, include
from django.conf.urls.static import static
from django.contrib import admin
from django.views.generic import TemplateView
from django.views.static import serve

DEBUG_ID = getattr(settings, 'DEBUG_ID', 'null')
DEBUG_REPORTING_UNIT = getattr(settings, 'DEBUG_REPORTING_UNIT', 'null')

class IndexView(TemplateView):
    template_name = 'index.html'

    def get_context_data(self, **kwargs):
        data = super().get_context_data(**kwargs)
        data['debug_id'] = DEBUG_ID
        data['debug_ru_id'] = DEBUG_REPORTING_UNIT
        return data

urlpatterns = [
    url(r'^admin/', admin.site.urls),
    url(r'^api/', include('landscapesim.urls')),
    url(r'^maps/', include('ncdjango.urls')),
    url(r'^$', IndexView.as_view()),
    url(r'^downloads/(?P<path>.*)$', serve, {'document_root': settings.DATASET_DOWNLOAD_DIR})
]

if settings.DEBUG:
    # Use Django to serve static files in development
    urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)


# coding=utf-8
import argparse

__author__ = 'ipetrash'

def create_parser():
    parse = argparse.ArgumentParser(description="Output a user-defined string in different encodings.")
    return parse

if __name__ == '__main__':
    create_parser().parse_args()

    string = raw_input("text = ")
    uni = unicode(string)

    list_encoding = {"windows-1251", "UTF-16", "UTF-16LE", "UTF-16BE", "ASCII", "Latin-1"}
    for encoding in list_encoding:
         print(encoding + ": " + uni.encode(encoding))