#!/usr/bin/env python
# -*- coding: utf-8 -*-
# by vellhe 2018/4/7
import math

from keras import Sequential, Input, Model
from keras.layers import Flatten, Dense, Dropout, Conv2D, Concatenate, \
    MaxPool2D
from keras.regularizers import l2

from base_model.base_model_creator import BaseModelCreator

class TextCNN(BaseModelCreator):
    '''
    TextCNN模型实现
    '''

    def __init__(self, config):
        self.config = dict()
        self.config["word_num"] = 50
        self.config["embedding_dim"] = 100
        self.config["channels"] = 1
        self.config["labels_num"] = 2

        self.config["filter_sizes"] = [3, 4, 5]
        self.config["pool_sizes"] = [48, 47, 46]  # default value: word_num - filter_size + 1
        self.config["num_filters"] = 128
        self.config["dropout"] = 0.5
        self.config["cnn_activation"] = "elu"
        self.config["dense_units"] = 1024
        self.config["dense_activation"] = "elu"
        self.config["weight_decay"] = 0

        self.config.update(config)
        super().__init__()

    def create(self, weights_path=None):
        '''
        channel_last (rows, cols, channels) <=> (word_num, embedding_dim, channels)
        :param weights_path:
        :return:
        '''
        word_num = self.config["word_num"]
        embedding_dim = self.config["embedding_dim"]
        channels = self.config["channels"]
        labels_num = self.config["labels_num"]

        filter_sizes = self.config["filter_sizes"]
        pool_sizes = self.config["pool_sizes"]
        num_filters = self.config["num_filters"]
        dropout = self.config["dropout"]
        dense_units = self.config["dense_units"]
        cnn_activation = self.config["cnn_activation"]
        dense_activation = self.config["dense_activation"]
        weight_decay = self.config["weight_decay"]

        # 文本内容卷积
        input_tensor = Input((word_num, embedding_dim, channels))

        conv_blocks = list()
        for idx, filter_size in enumerate(filter_sizes):
            conv = Conv2D(num_filters, kernel_size=(filter_size, embedding_dim), padding='valid',
                          kernel_initializer='normal',
                          kernel_regularizer=l2(weight_decay),
                          # activity_regularizer=regularizers.l1(weight_decay),
                          activation=cnn_activation)(input_tensor)

            maxpool = MaxPool2D(pool_size=(math.ceil(pool_sizes[idx]), 1),
                                strides=(int(pool_sizes[idx]), 1), padding='valid')(conv)
            conv_blocks.append(maxpool)

        concatenated_tensor = Concatenate(axis=1)(conv_blocks)
        flatten = Flatten()(concatenated_tensor)
        output_tensor = Dropout(dropout)(flatten)

        model_content = Model(inputs=input_tensor, outputs=output_tensor)

        # 分类
        model = Sequential()
        model.add(model_content)
        model.add(Dense(dense_units, activation=dense_activation))
        model.add(Dropout(dropout))
        model.add(Dense(labels_num, activation="softmax"))

        model_content.summary(positions=[.33, .60, .77, 1.])
        model.summary(positions=[.33, .60, .77, 1.])

        if weights_path:
            model.load_weights(weights_path)

        self.keras_model = model
        return self.keras_model


from csv import DictReader

from django.core.management.base import BaseCommand

from reviews.models import Category, Comment, Genre, GenreTitle, Review, Title
from users.models import User

FILES = [
    ('static/data/users.csv', User),
    ('static/data/category.csv', Category),
    ('static/data/genre.csv', Genre),
    ('static/data/genre_title.csv', GenreTitle)
]

class Command(BaseCommand):

    def reader(self, file, object):
        for row in DictReader(open(file, 'r', encoding='utf-8')):
            object(**row).save()

    def handle(self, *args, **options):
        for i in range(2):
            self.reader(*FILES[i])
        with open('static/data/titles.csv', 'r', encoding='utf-8') as inp_f:
            reader = DictReader(inp_f)
            for row in reader:
                titles = Title(
                    id=row['id'],
                    name=row['name'],
                    year=row['year'],
                    category=Category.objects.get(pk=row['category'])
                )
                titles.save()
        self.reader(*FILES[3])
        with open('static/data/review.csv', 'r', encoding='utf-8') as inp_f:
            reader = DictReader(inp_f)
            for row in reader:
                review = Review(
                    id=row['id'],
                    title_id=row['title_id'],
                    text=row['text'],
                    author_id=row['author'],
                    score=row['score'],
                    pub_date=row['pub_date'],
                )
                review.save()
        with open('static/data/comments.csv', 'r', encoding='utf-8') as inp_f:
            reader = DictReader(inp_f)
            for row in reader:
                comments = Comment(
                    id=row['id'],
                    review_id=row['review_id'],
                    text=row['text'],
                    author=User.objects.get(pk=row['author']),
                    pub_date=row['pub_date'],
                )
                comments.save()


# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Support for running jobs that are invoked via the `run-task` script.
"""

import os

from mozbuild.util import memoize
from mozpack import path
from taskgraph.util.schema import Schema
from taskgraph.util.yaml import load_yaml
from voluptuous import Any, Extra, Optional, Required

from gecko_taskgraph import GECKO
from gecko_taskgraph.transforms.job import run_job_using
from gecko_taskgraph.transforms.job.common import add_tooltool, support_vcs_checkout
from gecko_taskgraph.transforms.task import taskref_or_string

run_task_schema = Schema(
    {
        Required("using"): "run-task",
        # if true, add a cache at ~worker/.cache, which is where things like pip
        # tend to hide their caches.  This cache is never added for level-1 jobs.
        # TODO Once bug 1526028 is fixed, this and 'use-caches' should be merged.
        Required("cache-dotcache"): bool,
        # Whether or not to use caches.
        Optional("use-caches"): bool,
        # if true (the default), perform a checkout of gecko on the worker
        Required("checkout"): bool,
        Optional(
            "cwd",
            description="Path to run command in. If a checkout is present, the path "
            "to the checkout will be interpolated with the key `checkout`",
        ): str,
        # The sparse checkout profile to use. Value is the filename relative to
        # "sparse-profile-prefix" which defaults to "build/sparse-profiles/".
        Required("sparse-profile"): Any(str, None),
        # The relative path to the sparse profile.
        Optional("sparse-profile-prefix"): str,
        # if true, perform a checkout of a comm-central based branch inside the
        # gecko checkout
        Required("comm-checkout"): bool,
        # The command arguments to pass to the `run-task` script, after the
        # checkout arguments.  If a list, it will be passed directly; otherwise
        # it will be included in a single argument to `bash -cx`.
        Required("command"): Any([taskref_or_string], taskref_or_string),
        # Context to substitute into the command using format string
        # substitution (e.g {value}). This is useful if certain aspects of the
        # command need to be generated in transforms.
        Optional("command-context"): {
            # If present, loads a set of context variables from an unnested yaml
            # file. If a value is present in both the provided file and directly
            # in command-context, the latter will take priority.
            Optional("from-file"): str,
            Extra: object,
        },
        # Base work directory used to set up the task.
        Optional("workdir"): str,
        # If not false, tooltool downloads will be enabled via relengAPIProxy
        # for either just public files, or all files. Only supported on
        # docker-worker.
        Required("tooltool-downloads"): Any(
            False,
            "public",
            "internal",
        ),
        # Whether to run as root. (defaults to False)
        Optional("run-as-root"): bool,
    }
)

def common_setup(config, job, taskdesc, command):
    run = job["run"]
    if run["checkout"]:
        support_vcs_checkout(config, job, taskdesc, sparse=bool(run["sparse-profile"]))
        command.append(
            "--gecko-checkout={}".format(taskdesc["worker"]["env"]["GECKO_PATH"])
        )

    if run["sparse-profile"]:
        sparse_profile_prefix = run.pop(
            "sparse-profile-prefix", "build/sparse-profiles"
        )
        sparse_profile_path = path.join(sparse_profile_prefix, run["sparse-profile"])
        command.append(f"--gecko-sparse-profile={sparse_profile_path}")

    taskdesc["worker"].setdefault("env", {})["MOZ_SCM_LEVEL"] = config.params["level"]

worker_defaults = {
    "cache-dotcache": False,
    "checkout": True,
    "comm-checkout": False,
    "sparse-profile": None,
    "tooltool-downloads": False,
    "run-as-root": False,
}

load_yaml = memoize(load_yaml)

def script_url(config, script):
    if "MOZ_AUTOMATION" in os.environ and "TASK_ID" not in os.environ:
        raise Exception("TASK_ID must be defined to use run-task on generic-worker")
    task_id = os.environ.get("TASK_ID", "<TASK_ID>")
    tc_url = "http://firefox-ci-tc.services.mozilla.com"
    return f"{tc_url}/api/queue/v1/task/{task_id}/artifacts/public/{script}"

def substitute_command_context(command_context, command):
    from_file = command_context.pop("from-file", None)
    full_context = {}
    if from_file:
        full_context = load_yaml(os.path.join(GECKO, from_file))
    else:
        full_context = {}

    full_context.update(command_context)

    if isinstance(command, list):
        for i in range(len(command)):
            command[i] = command[i].format(**full_context)
    else:
        command = command.format(**full_context)

    return command

@run_job_using(
    "docker-worker", "run-task", schema=run_task_schema, defaults=worker_defaults
)
def docker_worker_run_task(config, job, taskdesc):
    run = job["run"]
    worker = taskdesc["worker"] = job["worker"]
    command = ["/builds/worker/bin/run-task"]
    common_setup(config, job, taskdesc, command)

    if run["tooltool-downloads"]:
        internal = run["tooltool-downloads"] == "internal"
        add_tooltool(config, job, taskdesc, internal=internal)

    if run.get("cache-dotcache"):
        worker["caches"].append(
            {
                "type": "persistent",
                "name": "{project}-dotcache".format(**config.params),
                "mount-point": "{workdir}/.cache".format(**run),
                "skip-untrusted": True,
            }
        )

    if run.get("command-context"):
        run_command = substitute_command_context(
            run.get("command-context"), run["command"]
        )
    else:
        run_command = run["command"]

    run_cwd = run.get("cwd")
    if run_cwd and run["checkout"]:
        run_cwd = path.normpath(
            run_cwd.format(checkout=taskdesc["worker"]["env"]["GECKO_PATH"])
        )
    elif run_cwd and "{checkout}" in run_cwd:
        raise Exception(
            "Found `{{checkout}}` interpolation in `cwd` for task {name} "
            "but the task doesn't have a checkout: {cwd}".format(
                cwd=run_cwd, name=job.get("name", job.get("label"))
            )
        )

    # dict is for the case of `{'task-reference': text_type}`.
    if isinstance(run_command, (str, dict)):
        run_command = ["bash", "-cx", run_command]
    if run["comm-checkout"]:
        command.append(
            "--comm-checkout={}/comm".format(taskdesc["worker"]["env"]["GECKO_PATH"])
        )
    if run["run-as-root"]:
        command.extend(("--user", "root", "--group", "root"))
    if run_cwd:
        command.extend(("--task-cwd", run_cwd))
    command.append("--")
    command.extend(run_command)
    worker["command"] = command

@run_job_using(
    "generic-worker", "run-task", schema=run_task_schema, defaults=worker_defaults
)
def generic_worker_run_task(config, job, taskdesc):
    run = job["run"]
    worker = taskdesc["worker"] = job["worker"]
    is_win = worker["os"] == "windows"
    is_mac = worker["os"] == "macosx"
    is_bitbar = worker["os"] == "linux-bitbar"

    if run["tooltool-downloads"]:
        internal = run["tooltool-downloads"] == "internal"
        add_tooltool(config, job, taskdesc, internal=internal)

    if is_win:
        command = ["C:/mozilla-build/python3/python3.exe", "run-task"]
    elif is_mac:
        command = ["/usr/local/bin/python3", "run-task"]
    else:
        command = ["./run-task"]

    common_setup(config, job, taskdesc, command)

    worker.setdefault("mounts", [])
    if run.get("cache-dotcache"):
        worker["mounts"].append(
            {
                "cache-name": "{project}-dotcache".format(**config.params),
                "directory": "{workdir}/.cache".format(**run),
            }
        )
    worker["mounts"].append(
        {
            "content": {
                "url": script_url(config, "run-task"),
            },
            "file": "./run-task",
        }
    )
    if job.get("fetches", {}):
        worker["mounts"].append(
            {
                "content": {
                    "url": script_url(config, "fetch-content"),
                },
                "file": "./fetch-content",
            }
        )

    run_command = run["command"]
    run_cwd = run.get("cwd")
    if run_cwd and run["checkout"]:
        run_cwd = path.normpath(
            run_cwd.format(checkout=taskdesc["worker"]["env"]["GECKO_PATH"])
        )
    elif run_cwd and "{checkout}" in run_cwd:
        raise Exception(
            "Found `{{checkout}}` interpolation in `cwd` for task {name} "
            "but the task doesn't have a checkout: {cwd}".format(
                cwd=run_cwd, name=job.get("name", job.get("label"))
            )
        )

    # dict is for the case of `{'task-reference': text_type}`.
    if isinstance(run_command, (str, dict)):
        if is_win:
            if isinstance(run_command, dict):
                for k in run_command.keys():
                    run_command[k] = f'"{run_command[k]}"'
            else:
                run_command = f'"{run_command}"'
        run_command = ["bash", "-cx", run_command]

    if run.get("command-context"):
        run_command = substitute_command_context(
            run.get("command-context"), run_command
        )

    if run["comm-checkout"]:
        command.append(
            "--comm-checkout={}/comm".format(taskdesc["worker"]["env"]["GECKO_PATH"])
        )

    if run["run-as-root"]:
        command.extend(("--user", "root", "--group", "root"))
    if run_cwd:
        command.extend(("--task-cwd", run_cwd))
    command.append("--")
    if is_bitbar:
        # Use the bitbar wrapper script which sets up the device and adb
        # environment variables
        command.append("/builds/taskcluster/script.py")
    command.extend(run_command)

    if is_win:
        taskref = False
        for c in command:
            if isinstance(c, dict):
                taskref = True

        if taskref:
            cmd = []
            for c in command:
                if isinstance(c, dict):
                    for v in c.values():
                        cmd.append(v)
                else:
                    cmd.append(c)
            worker["command"] = [{"artifact-reference": " ".join(cmd)}]
        else:
            worker["command"] = [" ".join(command)]
    else:
        worker["command"] = [
            ["chmod", "+x", "run-task"],
            command,
        ]


from Crypto.Cipher import AES
from Crypto.Util.Padding import unpad
import base64

def decrypt_aes_cbc(data, key, iv):
    cipher = AES.new(key, AES.MODE_CBC, iv)
    decrypted_data = cipher.decrypt(data)
    unpadded_data = unpad(decrypted_data, AES.block_size)
    return unpadded_data.decode('utf-8')

encrypted_data = "cEIZL9HlDGNhLVrygqnXFg=="  # Data terenkripsi dalam bentuk Base64
key = b'YOUR-KEY'  # Kunci enkripsi AES 128-bit (16 byte)
iv = b'YOUR-IV'  # Vektor inisialisasi (IV) 16-byte

encrypted_data = base64.b64decode(encrypted_data)

decrypted_data = decrypt_aes_cbc(encrypted_data, key, iv)
print("Data Terdekripsi:", decrypted_data)


"""N-Caltech101 Video maker.

This file makes Caltech101 Luma frames by using published
ncaltech_syn_images.

The data source is here:
https://github.com/uzh-rpg/rpg_vid2e

Each video has 161 frames, according to the provided timestamps,
the frame interval is:
298507462/160 = 1865671.6375 nanos

therefore, the intrinsic frame rate is:
1/(1865671.6375/1e9) = 536.000001233 FPS

The videos in the script are rendered at 30 FPS,
the input slowmotion factor is:
536.000001233/30 = 17.866666708 times

Author: Yuhuang Hu
Email : yuhuang.hu@ini.uzh.ch
"""

import argparse
import os
import glob

import subprocess

parser = argparse.ArgumentParser()

parser.add_argument("--data_root", type=str)
parser.add_argument("--output_root", type=str)

args = parser.parse_args()

valid_folders = sorted(
    glob.glob(
        os.path.join(args.data_root, "*", "image_*")))
valid_folders = [x for x in valid_folders if ".npz" not in x]

for folder in valid_folders:
    assert os.path.isdir(folder)

    video_base_name = os.path.basename(folder)
    parent_folder_name = os.path.dirname(folder)

    images_list = os.path.join(folder, "images", "*.png")

    # target output folder
    target_folder_name = parent_folder_name.replace(
        args.data_root, args.output_root)
    if not os.path.isdir(target_folder_name):
        os.makedirs(target_folder_name)

    target_file_name = os.path.join(
        target_folder_name, video_base_name+".mp4")

    # run subprocess to create videos
    subprocess.run(
        ["ffmpeg", "-framerate", "30",
         "-pattern_type", "glob", "-i", images_list,
         "-c:v", "libx265", "-x265-params", "lossless=1",
         target_file_name])

    print("Saving to {}".format(target_file_name))


import os
import inspect
import sys
from functools import wraps

this_script_path = os.path.dirname(
    os.path.abspath(__file__)
)

sys.path.append(os.path.join(this_script_path, '..', 'src'))

from src.logger import Logger

class BaseTest(object):

    @classmethod
    def _test_logger(cls, doc_text="General"):
        """
        Wrapper that writes a CSV formatted message to a log file for further processing
        :param doc_text: Colon separated text for document

        :return: None
        """
        def _test_logger_wrapper(func):
            @wraps(func)
            def wrapper(self, *args, **kwargs):
                func_name = func.__name__
                func_status = False
                output_msg = False
                try:
                    func(self, *args, **kwargs)
                    func_status = True
                    output_msg = True
                    self._write_logger(
                        doc_text=doc_text,
                        file_name=os.path.basename(inspect.getfile(func)),
                        func_name=func_name,
                        func_status=func_status)
                finally:
                    # prevent double logging output
                    if not output_msg:
                        self._write_logger(
                            doc_text=doc_text,
                            file_name=os.path.basename(inspect.getfile(func)),
                            func_name=func_name,
                            func_status=func_status)
            # make output the called function for unittest to work
            _test_logger_wrapper.__wrapped__ = func
            return wrapper
        return _test_logger_wrapper

    @staticmethod
    def _write_logger(
            doc_text,
            file_name,
            func_name,
            func_status,
            testing_logger=Logger(logger_name='testing_logger').logger):
        """
        Write a structured output to logging file.

        :param doc_text: Colon separated string indicating Document section and message
        :param file_name: Function file name
        :param func_name: Function name
        :param func_status: Final status of function
        :param testing_logger: logger object to write with
        :return: None
        """
        try:
            testing_logger.info(
                '%s,%s,%s,%s',
                doc_text,
                file_name,
                func_name,
                func_status)
        except AttributeError:
            pass
        return


from flask import jsonify,request
from helper.dbhelpers import run_query
from app import app

@app.post('/api/orders')
def orders_post():
    data = request.json
    is_canceled = data.get('is_canceled')
    is_confirmed = data.get('is_confirmed')
    is_completed = data.get('is_completed')
    client_id = data.get('client_id')
    restaurant_id = data.get('restaurant_id')

    if not is_confirmed:
        return jsonify("missing required arguement: is_confirmed "),422
    if not is_completed:
        return jsonify("missing required arguement: is_completed "),422
    if not is_canceled:
        return jsonify("missing required argument: is_canceled ")
    if not client_id:
        return jsonify("missing required arguement: client_id"),422
    if not restaurant_id:
        return jsonify('missing required argument: restaurant_id'),422

    #DB write
    run_query("INSERT INTO orders (is_confirmed, is_completed, is_canceled, client_id,restaurant_id ) VALUES (?,?,?,?,?)",
                [is_confirmed, is_completed,client_id ,restaurant_id])
    return jsonify("Post created sucsessfully!"),200

@app.get('/api/orders')
def orders_get():
    headers = request.headers
    tokens = headers.get("token")
    if not tokens :

        return jsonify("user is not authorized"),401

    checkuser = run_query("SELECT restaurant_id FROM restaurant_session WHERE token=?", [tokens])
    if checkuser == []:
        return jsonify("user does not have access!"),401
    client_id = checkuser[0][0]

    get_content = run_query("SELECT id, name, description, price, image_url, restaurant_id from menu WHERE id=?",[client_id])
    data = request.json
    restaruant_id = data.get('restaurant_id')
    client_id = data.get('client_id')

    get_content = run_query("SELECT * from orders")
    resp = []
    for content in get_content:
        obj ={}
        obj['id']= content[0]
        obj['is_confirmed']= content[1]
        obj['is_completed']= content[2]
        obj['is_canceled']= content[3]
        obj['client_id']= content[4]
        obj['restaurant_id']= content[5]
        resp.append(obj)
    if not get_content:
        return jsonify("Error , couldn't process get request!"),422
    if not restaruant_id:
        return jsonify("missing argument required: restaurant"),422
    if not client_id:
        return jsonify("missing arguement required: client_id"),422
    return jsonify(resp),200

@app.patch('/api/orders')
def orders_patch():
    headers = request.headers
    tokens = headers.get("token")
    if not tokens :

        return jsonify("user is not authorized"),401

    checkuser = run_query("SELECT restaurant_id FROM restaurant_session WHERE token=?", [tokens])
    if checkuser == []:
        return jsonify("user does not have access!"),401
    client_id = checkuser[0][0]

    run_query("SELECT id, name, description, price, image_url, restaurant_id from menu WHERE id=?",[client_id])
    data = request.json
    is_canceled = data.get('is_canceled')
    is_completed = data.get('is_completed')
    is_confirmed = data.get('is_confirmed')

    if not is_canceled:
        return jsonify("missing required arguement: price "),422
    if not is_completed:
        return jsonify("missing required arguement: name "),422
    if not is_confirmed :
        return jsonify("missing required arguement: description"),422

    #db write
    return jsonify("Updated sucsessfully!"),200


import time
import logging

import tenacity
from tenacity.retry import retry_if_result, retry_if_exception_type
from tenacity import RetryError

LOGGER = logging.getLogger('sdcm.wait')

def wait_for(func, step=1, text=None, timeout=None, throw_exc=True, **kwargs):
    """
    Wrapper function to wait with timeout option.

    :param func: Function to evaluate.
    :param step: Time to sleep between attempts in seconds
    :param text: Text to print while waiting, for debug purposes
    :param timeout: Timeout in seconds
    :param throw_exc: Raise exception if timeout expired, but func result is not True
    :param kwargs: Keyword arguments to func
    :return: Return value of func.
    """
    if not timeout:
        return forever_wait_for(func, step, text, **kwargs)

    res = None

    def retry_logger(retry_state):
        # pylint: disable=protected-access
        LOGGER.debug(
            'wait_for: Retrying %s: attempt %s ended with: %s',
            text or retry_state.fn.__name__,
            retry_state.attempt_number,
            retry_state.outcome._exception or retry_state.outcome._result,
        )

    try:
        retry = tenacity.Retrying(
            reraise=throw_exc,
            stop=tenacity.stop_after_delay(timeout),
            wait=tenacity.wait_fixed(step),
            before_sleep=retry_logger,
            retry=(retry_if_result(lambda value: not value) | retry_if_exception_type())
        )
        res = retry(func, **kwargs)

    except Exception as ex:  # pylint: disable=broad-except
        err = f"Wait for: {text or func.__name__}: timeout - {timeout} seconds - expired"
        LOGGER.error(err)
        if hasattr(ex, 'last_attempt') and ex.last_attempt.exception() is not None:  # pylint: disable=no-member
            LOGGER.error("last error: %r", ex.last_attempt.exception())  # pylint: disable=no-member
        else:
            LOGGER.error("last error: %r", ex)
        if throw_exc:
            if hasattr(ex, 'last_attempt') and not ex.last_attempt._result:  # pylint: disable=protected-access,no-member
                raise RetryError(err) from ex
            raise

    return res

def forever_wait_for(func, step=1, text=None, **kwargs):
    """
    Wait indefinitely until func evaluates to True.

    This is similar to avocado.utils.wait.wait(), but there's no
    timeout, we'll just keep waiting for it.

    :param func: Function to evaluate.
    :param step: Amount of time to sleep before another try.
    :param text: Text to log, for debugging purposes.
    :param kwargs: Keyword arguments to func
    :return: Return value of func.
    """
    ok = False
    start_time = time.time()
    while not ok:
        ok = func(**kwargs)
        time.sleep(step)
        time_elapsed = time.time() - start_time
        if text is not None:
            LOGGER.debug('%s (%s s)', text, time_elapsed)
    return ok


import cv2
import numpy as np
import os

path = os.path.dirname(os.path.abspath(__file__))

#treinando o algoritmo de reconhecimento

recognizer = cv2.face.LBPHFaceRecognizer_create()

recognizer.read(f'{path}/trainer/trainer.yml')

cascadePath = cv2.data.haarcascades + "haarcascade_frontalface_default.xml"
faceCascade = cv2.CascadeClassifier(cascadePath)
font = cv2.FONT_HERSHEY_SIMPLEX

id = 0

names = []

f = open(f'{path}/names.txt', 'r')
for line in f:
    names.append(line.split('-')[1].strip())

print(names)

cam = cv2.VideoCapture(0)
cam.set(3, 800)
cam.set(4, 600)

#Definir o tamanho minimo da janela a ser reconhecido como uma face
minW = 0.1*cam.get(3)
minH = 0.1*cam.get(4)

while True:
    ret, img = cam.read()
    img = cv2.flip(img, 1)
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

    faces = faceCascade.detectMultiScale(
        gray,
        scaleFactor = 1.2,
        minNeighbors = 2,
        minSize = (int(minW), int(minH))
    )
    for(x, y, w, h) in faces:
        id, confidence = recognizer.predict(gray[y:y+h, x:x+w])

        if confidence > 50:
            cv2.rectangle(img, (x, y), (x+w, y+h), (0, 255, 0), 2)

            ########################

            id = names[id]
            confidence = " {0}%".format(round(confidence))
        else:
            cv2.rectangle(img, (x, y), (x+w, y+h), (0, 0, 255), 2)
            id = "Sem registro"
            confidence = " {0}%".format(round(confidence))

        cv2.putText(
            img,
            str(id),
            (x+5, y-5),
            font,
            1,
            (255, 255, 255),
            2
        )
        cv2.putText(
            img,
            str(confidence),
            (x + 5, y + h-5),
            font,
            1,
            (255, 255, 0),
            1
        )

    cv2.imshow('camera', img)
    k = cv2.waitKey(10) & 0xff
    if k == 27:
        break

print("\n Saindo do programa e limpando")
cam.release()
cv2.destroyAllWindows()


import tensorflow as tf
import numpy as np
from vizdoom import *

import random
import time
from skimage import transform
from collections import deque
import matplotlib.pyplot as plt

import warnings # This ignore all the warning messages that are normally printed during the training because of skiimage
warnings.filterwarnings('ignore')

def create_environment():
    game = DoomGame()

    # Richtige Spielconfig laden
    game.load_config("basic.cfg")

    # Richtiges Spielszenario laden
    game.set_doom_scenario_path("basic.wad")

    game.init()

    # Mögliche Actionen
    left = [1, 0, 0]
    right = [0, 1, 0]
    shoot = [0, 0, 1]
    possible_actions = [left, right, shoot]

    return game, possible_actions

def test_environment():
    game = DoomGame()
    game.load_config("basic.cfg")
    game.set_doom_scenario_path("basic.wad")
    game.init()
    left = [1, 0, 0]
    right = [0, 1, 0]
    shoot = [0, 0, 1]
    possible_actions = [left, right, shoot]
    episodes = 10

    for i in range(episodes):
        print("New episode started")
        game.new_episode()
        step = 0
        while not game.is_episode_finished():
            state = game.get_state()
            img = state.screen_buffer
            misc = state.game_variables
            action = random.choice(possible_actions)
            print(action)
            reward = game.make_action(action)
            step += 1
            if step == 30:
                break
            print("\treward:", reward)
            # time.sleep(2)
        print("Result:", game.get_total_reward())
        time.sleep(2)
    game.close()

def preprocess_frame(frame):
    # Graustufe muss nicht erstellt werden, da Ausgabe für Script schon grau.
    #x = np.mean(frame, -1)
    cropped_frame = frame[30:-10,30:-30]
    normalized_frame = cropped_frame/255.0
    preprocessed_frame = transform.resize(normalized_frame, [84, 84])

    return preprocessed_frame

def stack_frames(stacked_frames, state, is_new_episode):
    frame = preprocess_frame(state)

    if is_new_episode:
        stacked_frames = deque([np.zeros((84, 84), dtype=np.int) for i in range(stack_size)], maxlen=4)
        stacked_frames.append(frame)
        stacked_frames.append(frame)
        stacked_frames.append(frame)
        stacked_frames.append(frame)
        stacked_state = np.stack(stacked_frames, axis=2)

    else:
        stacked_frames.append(frame)

        stacked_state = np.stack(stacked_frames, axis=2)

    return stacked_state, stacked_frames
# test_environment()
game, possible_actions = create_environment()

stack_size = 4
stacked_frames = deque([np.zeros((84,84), dtype=np.int) for i in range(stack_size)], maxlen=4)

### Modelhyperparameter
state_size = [84, 84, 4]
action_size = game.get_available_buttons_size()
learning_rate = 0.0002

### Training Hyperparameter
total_episodes = 5
max_steps = 100
batch_size = 64

### Exploration Parameter
explore_start = 1.0
explore_stop = 0.01
decay_rate = 0.0001

# Q Learning Parameter
gamma = 0.95

# Memory Hyperparameter
pretrain_lenght = batch_size
memory_size = 1000000

### Trained Agent
training = True

### Anschalten, wenn das Env gerendert werden soll
episode_render = False

class DQNetwork:
    def __init__(self, state_size, action_size, learning_rate, name='DQNetwork'):
        self.state_size = state_size
        self.action_size = action_size
        self. learning_rate = learning_rate

        with tf.variable_scope(name):
            # Platzhalter erstellen
            self.inputs_ = tf.placeholder(tf.float32, [None, *state_size], name="inputs")
            self.actions_ = tf.placeholder(tf.float32, [None, 3], name="actions")

            self.target_Q = tf.placeholder(tf.float32, [None], name="target")

            """ 1. Convolutional Layer"""
            # Input 84x84x4
            self.conv1 = tf.layers.conv2d(
                inputs= self.inputs_,
                filters = 32,
                kernel_size = [8, 8],
                strides = [4, 4],
                padding = "VALID",
                kernel_initializer = tf.contrib.layers.xavier_initializer_conv2d(),
                name = "conv1"
                )

            self.conv1_batchnorm = tf.layers.batch_normalization(
                self.conv1,
                training = True,
                epsilon = 1e-5,
                name = "batch_norm1"
                )

            self.conv1_out = tf.nn.elu(
                self.conv1_batchnorm,
                name = "conv1_out"
                )
            # --> [20, 20, 32]

            """2. Convolutional Layer"""
            self.conv2 = tf.layers.conv2d(
                inputs= self.conv1_out,
                filters = 64,
                kernel_size = [4, 4],
                strides = [2, 2],
                padding = "VALID",
                kernel_initializer = tf.contrib.layers.xavier_initializer_conv2d(),
                name = "conv2"
                )

            self.conv2_batchnorm = tf.layers.batch_normalization(
                self.conv2,
                training = True,
                epsilon = 1e-5,
                name = "batch_norm2"
                )

            self.conv2_out = tf.nn.elu(
                self.conv2_batchnorm,
                name = "conv2_out"
                )
            # --> [9, 9, 64]

            """3. Convolutional Layer"""
            self.conv3 = tf.layers.conv2d(
                inputs= self.conv2_out,
                filters = 128,
                kernel_size = [4, 4],
                strides = [2, 2],
                padding = "VALID",
                kernel_initializer = tf.contrib.layers.xavier_initializer_conv2d(),
                name = "conv3"
                )

            self.conv3_batchnorm = tf.layers.batch_normalization(
                self.conv3,
                training = True,
                epsilon = 1e-5,
                name = "batch_norm3"
                )

            self.conv3_out = tf.nn.elu(
                self.conv3_batchnorm,
                name = "conv3_out"
                )
            # --> [3, 3, 128]

            """Flatten Layer"""

            self.flatten = tf.layers.flatten(
                self.conv3_out
                )
            # --> [1152]

            """Fully Connected Layer"""
            self.fc = tf.layers.dense(
                inputs = self.flatten,
                units = 512,
                activation = tf.nn.elu,
                kernel_initializer = tf.contrib.layers.xavier_initializer(),
                name="fc1"
                )

            self.output = tf.layers.dense(
                inputs = self.fc,
                kernel_initializer = tf.contrib.layers.xavier_initializer(),
                units = 3,
                activation = None
                )

            # Q ist der verhergesagte Q Wert

            self.Q = tf.reduce_sum(tf.multiply(self.output, self.actions_), axis = 1)

            # Loss zwischen vorhergesagtem Q und Q Target

            self.loss = tf.reduce_mean(tf.square(self.target_Q - self.Q))

            self.optimizer = tf.train.RMSPropOptimizer(self.learning_rate).minimize(self.loss)

tf.reset_default_graph()
DQNetwork = DQNetwork(state_size, action_size, learning_rate)

class Memory():
    def __init__(self, max_size):
        self.buffer = deque(maxlen = max_size)

    def add(self, experience):
        self.buffer.append(experience)

    def sample(self, batch_size):
        buffer_size = len(self.buffer)
        index = np.random.choice(np.arange(buffer_size), size = batch_size, replace = False)
        return [self.buffer[i] for i in index]

memory = Memory(max_size = memory_size)

game.new_episode()

for i in range(pretrain_lenght):
    if i == 0:
        state = game.get_state().screen_buffer
        state, stacked_frames = stack_frames(stacked_frames, state, True)

    # Random Action
    action = random.choice(possible_actions)

    # Get the rewards
    reward = game.make_action(action)

    # Look if episode is finished
    done = game.is_episode_finished()

    if done:
        next_state = np.zeros(state.shape)
        memory.add((state, action, reward,next_state, done))
        game.new_episode()
        state = game.get_state().screen_buffer
        state, stacked_frames = stack_frames(stacked_frames, state, True)
    else:
        next_state = game.get_state().screen_buffer
        next_state, stacked_frames = stack_frames(stacked_frames, next_state, False)

        memory.add((state, action, reward, next_state, done))
        state = next_state

writer = tf.summary.FileWriter("./tensorboard/dqn/2")

tf.summary.scalar("Loss", DQNetwork.loss)

write_op = tf.summary.merge_all()

def predict_action(explore_start, explore_stop, decay_rate, decay_step, state, actions):
    exp_exp_tradeoff = np.random.rand()

    explore_probability = explore_stop + (explore_start - explore_stop) * np.exp(-decay_rate * decay_step)

    if (explore_probability > exp_exp_tradeoff):
        action = random.choice(possible_actions)
    else:
        Qs = sess.run(DQNetwork.output, feed_dict = {DQNetwork.inputs_: state.reshape((1, *state.shape))})

        choice = np.argmax(Qs)
        action = possible_actions[int(choice)]
    return action, explore_probability

saver = tf.train.Saver()

if training == True:
    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())

        decay_step = 0

        game.init()

        for episode in range(total_episodes):
            step = 0
            episode_rewards = []

            game.new_episode()
            state = game.get_state().screen_buffer

            state, stacked_frames = stack_frames(stacked_frames, state, True)

            while step < max_steps:
                step += 1
                decay_step += 1
                action, explore_probability = predict_action(explore_start, explore_stop, decay_rate, decay_step, state, possible_actions)

                reward = game.make_action(action)
                done = game.is_episode_finished()
                episode_rewards.append(reward)

                if done:
                    next_state = np.zeros((84, 84), dtype=np.int)
                    next_state, stacked_frames = stack_frames(stacked_frames, next_state, False)

                    step = max_steps
                    total_reward = np.sum(episode_rewards)

                    print(
                        'Episode: {}'.format(episode),
                        'Total reward: {}'.format(total_reward),
                        'Training Loss: {:.4f}'.format(loss),
                        'Explore P: {:.4f}'.format(explore_probability)
                        )
                    memory.add((state, action, reward, next_state, done))

                else:
                    next_state = game.get_state().screen_buffer
                    next_state, stacked_frames = stack_frames(stacked_frames, next_state, False)

                    memory.add((state, action, reward, next_state, done))

                    state = next_state

                batch = memory.sample(batch_size)
                states_mb = np.array([each[0] for each in batch], ndmin=3)
                actions_mb = np.array([each[1] for each in batch])
                rewards_mb = np.array([each[2] for each in batch])
                next_states_mb = np.array([each[3] for each in batch], ndmin=3)
                dones_mb = np.array([each[4] for each in batch])

                target_Qs_batch = []

                Qs_next_state = sess.run(DQNetwork.output, feed_dict = {DQNetwork.inputs_: next_states_mb})

                for i in range(len(batch)):
                    terminal = dones_mb[i]

                    if terminal:
                        target_Qs_batch.append(rewards_mb[i])
                    else:
                        target = rewards_mb[i] + gamma * np.max(Qs_next_state[i])
                        target_Qs_batch.append(target)

                targets_mb = np.array([each for each in target_Qs_batch])

                loss, _ = sess.run([DQNetwork.loss, DQNetwork.optimizer], feed_dict={DQNetwork.inputs_: states_mb,
                                                                                DQNetwork.target_Q: targets_mb,
                                                                                DQNetwork.actions_: actions_mb})
                summary = sess.run(write_op, feed_dict={DQNetwork.inputs_: states_mb,
                                                        DQNetwork.target_Q: targets_mb,
                                                        DQNetwork.actions_: actions_mb})
                writer.add_summary(summary, episode)
                writer.flush()

            # Modell alle 5 Episoden speichern
            if episode % 5 == 0:
                try:
                    save_path = saver.save(sess, "./models1/model.ckpt")
                    print("Model saved")
                except:
                    print("Saving skipped")

with tf.Session() as sess:
    game, possible_actions = create_environment()

    total_score = 0

    saver.restore(sess, "./models1/model.ckpt")
    game.init()

    for i in range(3):
        done = False
        game.new_episode()

        state = game.get_state().screen_buffer
        state, stacked_frames = stack_frames(stacked_frames, state, True)

        while not game.is_episode_finished():
            time.sleep(1)

            Qs = sess.run(DQNetwork.output, feed_dict={DQNetwork.inputs_: state.reshape((1, *state.shape))})

            choice = np.argmax(Qs)
            action = possible_actions[int(choice)]

            game.make_action(action)
            done = game.is_episode_finished()
            score = game.get_total_reward()

            if done:
               break
            else:
               print("playing")
               next_state = game.get_state().screen_buffer
               next_state, stacked_frames = stack_frames(stacked_frames, next_state, False)
               state = next_state
        score = game.get_total_reward()
        print("Score: ", score)

    time.sleep(5)
    game.close()


import pygame
from pygame.sprite import Sprite
import funcs as funcs

def load_image(name):
	image = pygame.image.load(funcs.find_data_file(name))
	return image

class ExplosionSprite(Sprite):
	def __init__(self, screen, pos, expl_type="plane", player=None):
		super().__init__()

		self.expl_type = expl_type

		if not player == None:
			self.my_creator = player

		self.images = []

		if self.expl_type == "plane":
			self.images.append(load_image('regularExplosion00.png'))
			self.images.append(load_image('regularExplosion01.png'))
			self.images.append(load_image('regularExplosion02.png'))
			self.images.append(load_image('regularExplosion03.png'))
			self.images.append(load_image('regularExplosion04.png'))
			self.images.append(load_image('regularExplosion05.png'))
			self.images.append(load_image('regularExplosion06.png'))
			self.images.append(load_image('regularExplosion07.png'))
			self.images.append(load_image('regularExplosion08.png'))

			self.rect = pygame.Rect(pos.centerx - 92, pos.centery - 92, 192, 192)
		else:
			pic0 = load_image('regularExplosion00.png')
			pic0 = pygame.transform.scale(pic0, (300, 300))
			self.images.append(pic0)

			pic1 = load_image('regularExplosion01.png')
			pic1 = pygame.transform.scale(pic1, (300, 300))
			self.images.append(pic1)

			pic2 = load_image('regularExplosion02.png')
			pic2 = pygame.transform.scale(pic2, (300, 300))
			self.images.append(pic2)

			pic3 = load_image('regularExplosion03.png')
			pic3 = pygame.transform.scale(pic3, (300, 300))
			self.images.append(pic3)

			pic4 = load_image('regularExplosion04.png')
			pic4 = pygame.transform.scale(pic4, (300, 300))
			self.images.append(pic4)

			pic5 = load_image('regularExplosion05.png')
			pic5 = pygame.transform.scale(pic5, (300, 300))
			self.images.append(pic5)

			pic6 = load_image('regularExplosion06.png')
			pic6 = pygame.transform.scale(pic6, (300, 300))
			self.images.append(pic6)

			pic7 = load_image('regularExplosion07.png')
			pic7 = pygame.transform.scale(pic7, (300, 300))
			self.images.append(pic7)

			pic8 = load_image('regularExplosion08.png')
			pic8 = pygame.transform.scale(pic8, (300, 300))
			self.images.append(pic8)

			self.rect = pygame.Rect(pos.centerx-150, pos.centery - 150, 300, 300)

		self.index = 0
		self.image = self.images[self.index]

		self.animation_time = 0.1
		self.current_time = 0
		self.animation_frames = 10
		self.current_frame = 0

	def update(self, dt):

		self.current_time += dt
		if self.current_time >= self.animation_time:
			self.index += 1
			if self.index >= len(self.images):
				self.index = 0
			self.image = self.images[self.index]

		""" self.current_frame += 1
		if self.current_frame >= self.animation_frames:
			self.current_frame = 0
			self.index = (self.index + 1) % len(self.images)
			self.image = self.images[self.index] """


#!/usr/bin/env python
# coding: utf-8

class Solution(object):
    def findDuplicate(self, paths):
        """
        :type paths: List[str]
        :rtype: List[List[str]]
        """
        from collections import defaultdict
        hash = defaultdict(list)
        for path in paths:
            dirname, *files = path.split()
            for file in files:
                filename, content = file[:-1].split('(', 1)
                hash[content].append(dirname + '/' + filename)
        return [fs for fs in hash.values() if len(fs) > 1]

if __name__ == '__main__':
    sol = Solution()
    print(sol.findDuplicate([]))
    print(sol.findDuplicate(
        ["root/a 1.txt(abcd) 2.txt(efgh)", "root/c 3.txt(abcd)",
         "root/c/d 4.txt(efgh)", "root 4.txt(efgh)"]
    ))


from . import TrackingService

import re
from datetime import datetime

class SwisspostChService(TrackingService):
    """ Swiss Post """

    name = "SwissPost.ch"
    url = 'https://www.post.ch/EasyTrack/submitParcelData.do?p_language=en&VTI-GROUP=1&directSearch=false&formattedParcelCodes=%(number)s'

    def _parse_page(self, html):
        res = re.search(r'
[\s\n\r]*(
[\s\n\r]*.*?)
', html, re.DOTALL)
        if res is None:
            return []
        html = res.group(1)
        results = []
        for res in re.findall(r'<span.*?(.*?)	.*?(.*?)	.*?(.*?)	.*?(.*?)	', html, re.DOTALL):
            resdate = re.sub('<([^ ]*)[^>]*>.*?</\1>', '', res[0].strip() + " " + res[1].strip())
            resdate = datetime.strptime(resdate, '%d.%m.%Y %H:%M')
            operation = re.sub('<([^ ]*)[^>]*>.*?</\\1>', '', res[2])
            results.append((operation, resdate, res[3].strip()))
        return results


import pytest

from app import db, models
from tests.fixtures.bot_users import get_bot_users

@pytest.mark.usefixtures('schedules')
class TestSchedule:
    def test_count_schedule(self, schedules):
        assert len(
            schedules) == 4572, 'В таблице с расписанием должно быть 4572 записи.'

    def test_schedule_type(self, schedules):
        assert all(type(schedule) == models.Schedule for schedule in schedules), (
                'Объекты должны быть экземплярами класса Schedule.'
        )

    @pytest.mark.asyncio
    async def test_select_schedule(self):
        schedules = await db.select_schedule(from_station_id=1, to_station_id=9)

        assert type(schedules) == list
        assert all(type(schedule) == models.Schedule for schedule in schedules), (
                'Объекты должны быть экземплярами класса Schedule.'
        )

class TestBotUser:
    @pytest.mark.asyncio
    async def test_insert_user(self, new_telegram_user):
        bot_users = get_bot_users()
        assert len(bot_users) == 0, 'Изначально таблица bot_user должна быть пустая.'

        await db.insert_user(new_telegram_user)

        bot_users = get_bot_users()
        assert len(bot_users) == 1, 'В таблице bot_user не появился новый пользователь.'
        assert bot_users[0].bot_user_id == new_telegram_user.id
        assert bot_users[0].first_name == new_telegram_user.first_name
        assert bot_users[0].last_name == new_telegram_user.last_name
        assert bot_users[0].username == new_telegram_user.username
        assert bot_users[0].is_bot == new_telegram_user.is_bot


import numpy as np
import time
import pycuda.autoinit
from config_pso import config_pso
import pycuda.driver as cuda
from pycuda.compiler import SourceModule
from pycuda.driver import cudaMemcpy, cudaMemcpyHostToDevice

mod = SourceModule("""
    __global__ void update_particles(float *particles, float *velocities, float *best_positions, float *global_best_position, float w, float c1, float c2, int nb_part, int dim)
    {
        int idx = blockIdx.x * blockDim.x + threadIdx.x;
        if (idx < nb_part * dim) {
            int i = idx / dim;
            int j = idx % dim;
            float rp = (float)rand() / (float)RAND_MAX;
            float rg = (float)rand() / (float)RAND_MAX;
            velocities[idx] = w * velocities[idx] + c1 * rp * (best_positions[idx] - particles[idx]) + c2 * rg * (global_best_position[j] - particles[idx]);
            particles[idx] += velocities[idx];
        }
    }
""")

update_particles = mod.get_function("update_particles")

def pso_opti_gpu(fct, params):
    begin = time.time()

    nb_part = params["nb_part"]
    dim = params["Dim"]
    nb_iter = params["max_ite"]
    nb_block = 128
    nb_thread = 512

    particles = np.random.uniform(params["min_x"], params["max_x"], (nb_part, dim)).astype(np.float32)
    velocities = np.random.uniform(-params["vit"], params["vit"], (nb_part, dim)).astype(np.float32)
    best_positions = np.copy(particles)
    costs = np.array([fct(particles[i]) for i in range(nb_part)], dtype=np.float32)
    best_costs = np.copy(costs)
    global_best_index = np.argmax(costs)
    global_best_cost = costs[global_best_index]
    global_best_position = particles[global_best_index].astype(np.float32)

    particles_gpu = cuda.to_device(particles)
    velocities_gpu = cuda.to_device(velocities)
    best_positions_gpu = cuda.to_device(best_positions)
    global_best_position_gpu = cuda.to_device(global_best_position)

    for _ in range(nb_iter):
        update_particles(
            particles_gpu, velocities_gpu, best_positions_gpu, global_best_position_gpu,
            np.float32(params["w"]), np.float32(params["c1"]), np.float32(params["c2"]),
            np.int32(nb_part), np.int32(dim),
            block=(nb_block, 1, 1), grid=(nb_thread, 1, 1)
        )

        cuda.memcpy_dtoh(particles, particles_gpu)

        costs = np.array([fct(particles[i]) for i in range(nb_part)], dtype=np.float32)
        improved_mask = costs > best_costs
        best_positions[improved_mask] = particles[improved_mask]
        best_costs[improved_mask] = costs[improved_mask]

        global_best_index = np.argmax(costs)
        new_global_best_cost = costs[global_best_index]
        if new_global_best_cost > global_best_cost:
            global_best_cost = new_global_best_cost
            global_best_position = particles[global_best_index].astype(np.float32)
            cudaMemcpy(global_best_position_gpu, global_best_position, global_best_position.nbytes, cudaMemcpyHostToDevice)

    end = time.time()
    print("Global best cost: ", global_best_cost)
    print("Global best position: ", global_best_position)
    print("Execution time: ", end - begin)

if __name__ == "__main__":

    params = config_pso()

    pso_opti_gpu(params['function'], params)


from PyQt5.QtGui import QPen, QColor
from PyQt5.QtCore import Qt

class Synapse:
    def __init__(self, from_neuron, to_neuron, weight, color=QColor(0, 0, 255)):
        self.from_neuron = from_neuron
        self.to_neuron = to_neuron
        self.weight = weight
        self.color = color
        self.width = weight
        self.set_width(weight)

    def draw(self, painter):
        pen = QPen(self.color)
        pen.setWidth(self.width);
        painter.setPen(pen);
        painter.drawLine(*self.from_neuron.pos, *self.to_neuron.pos)

    def update_weight(self, new_weight):
        self.weight = new_weight
        self.set_width(new_weight)

    def set_width(self, weight):
        self.width = abs(weight) / 2


import pygame.sprite

from figure import FigurePink
from jumping_bar import JumpingBar
from platform import Platform
from platform import Ground
from settings import Settings
import os, os.path

sets = Settings()

# Generate assets of the game
def generate_gd(settings):
    ground = JumpingBar([0, settings.gd_height],
                        size=[settings.can_w, settings.gd_thick],
                        color=settings.gd_color)

    return ground

def generate_gd_img(settings):
    bg = pygame.image.load("assets/bg.png")
    return bg

def generate_platforms(settings):
    group = pygame.sprite.Group()
    directory = os.getcwd()
    file_path = directory + '/assets/platforms/'
    file_names = [file_path + name for name in os.listdir(directory + '/assets/platforms/')]
    # Generate ground
    ground_path = directory + '/assets/platforms/ground.png'
    group.add(Ground(ground_path, [0, 0], settings))
    # Place the bar
    for i in range(len(file_names)):
        x = 50 + i * 150
        y = 300
        print("filename = ", file_names[i])
        platform = Platform(file_names[i], [x, y], settings)
        group.add(platform)
    return group

def generate_player(settings):
    # initial height = can_h - ground_h - figure_h
    init_location = [20, 0]
    return FigurePink(pos=init_location,
                      settings=settings)

def draw_platforms(win, group, settings):
    for platform in group:
        win.blit(platform.image, platform.drawing_pos)

def draw_player(win, player):
    win.blit(player.image, player.drawing_pos)

def locate(loc, ground_height, settings):
    '''Transform the location from left-bottom to left up.'''
    return loc[0], settings.can_h - ground_height - loc[1]

def screen(figure, settings, platforms, background):
    if figure.x < 200:
        settings.screen_x = 0
    else:
        settings.screen_x = figure.x - 200
    figure.drawing_pos = [figure.x - settings.screen_x, figure.y]
    for platform in platforms:
        platform.drawing_pos[0] = platform.x - settings.screen_x
    background.drawing_pos[0] = -settings.screen_x


import torch
import numpy as np
import sys
import gym

sys.path.append("./")
from base_net.model import *
from torch import nn, optim
import torch.nn.functional as F
from torch.distributions import Categorical

class TD3(nn.Module):
    def __init__(self, args):
        super(TD3, self).__init__()
        self.input_size, self.output_size, self.clamp, self.action_max, self.action_min, self.mem_size, self.toi, self.device, self.actor_lr, self.critic_lr = args
        self.critic_update_cnt = 0

        self.actor = Policy_net(args = (self.input_size, self.output_size))
        self.critic1 = Q_net(args = (self.input_size + self.output_size, 1))
        self.critic2 = Q_net(args = (self.input_size + self.output_size, 1))

        self.target_actor = Policy_net(args = (self.input_size, self.output_size))
        self.target_critic1 = Q_net(args = (self.input_size + self.output_size, 1))
        self.target_critic2 = Q_net(args = (self.input_size + self.output_size, 1))

        self.replay_buffer = ReplayBuffer(args = (self.mem_size))
        self.optimizer_actor = optim.Adam(self.actor.parameters(), lr = self.actor_lr)
        self.optimizer_critic = optim.Adam(self.parameters(), lr = self.critic_lr)
        self.update_target_net(initialize = True)

    def update_target_net(self, initialize = False):
        if initialize:
            self.target_actor.load_state_dict(self.actor.state_dict())
            self.target_critic1.load_state_dict(self.critic1.state_dict())
            self.target_critic2.load_state_dict(self.critic2.state_dict())

        elif not initialize:
            for raw, target in zip(self.actor.parameters(), self.target_actor.parameters()):
                target.data.copy_(self.toi * raw.data + (1 - self.toi) * target.data)

            for raw, target in zip(self.critic1.parameters(), self.target_critic1.parameters()):
                target.data.copy_(self.toi * raw.data + (1 - self.toi) * target.data)

            for raw, target in zip(self.critic2.parameters(), self.target_critic2.parameters()):
                target.data.copy_(self.toi * raw.data + (1 - self.toi) * target.data)

    def get_policy(self, inputs):
        policy_op = self.actor(inputs)
        action_op = torch.tanh(policy_op) * (self.action_max - self.action_min) / 2
        if self.clamp:
            action_op = torch.clamp(action_op, self.action_min, self.action_max)
        return action_op

    def get_target_policy(self, inputs):
        policy_op = self.actor(inputs)
        action_op = torch.tanh(policy_op) * (self.action_max - self.action_min) / 2
        if self.clamp:
            action_op = torch.clamp(action_op, self.action_min, self.action_max)
        return action_op

    def save_trans(self, transition):
        self.replay_buffer.save_trans(transition)

    def to_tensor(self, items):
        s, a, r, s_next, done = items
        s = torch.FloatTensor(s).to(self.device)
        a = torch.FloatTensor(a).to(self.device)
        r = torch.FloatTensor(r).to(self.device)
        s_next = torch.FloatTensor(s_next).to(self.device)
        done = torch.FloatTensor(done).to(self.device)

        return s, a, r.unsqueeze(-1), s_next, done.unsqueeze(-1)

    def select_action(self, inputs, epsilon, eval_mode = False):
        action_op = self.get_policy(inputs)
        noise = torch.randn(action_op.shape).to(self.device) if not eval_mode else 0.
        # action_op += noise * epsilon
        if self.clamp:
            action = torch.clamp(action_op, self.action_min, self.action_max)
        return action.detach().cpu().numpy()

    def train(self, gamma = 0.98, batch_size = 32, policy_update_interval = 50):
        s, a, r, s_next, done = self.to_tensor(self.replay_buffer.sample_batch(batch_size = batch_size))
        q_val1 = self.critic1(torch.cat([s, a], -1))
        q_val2 = self.critic2(torch.cat([s, a], -1))
        action_next = self.get_target_policy(s_next).detach()
        target_q_val = r + gamma * torch.min(self.target_critic1(torch.cat([s_next, action_next], -1)), self.target_critic2(torch.cat([s_next, action_next], -1))) * (1 - done)
        loss_critic = ((target_q_val.detach() - q_val1) ** 2 + (target_q_val.detach() - q_val2) ** 2).mean()

        if (self.critic_update_cnt + 1) % policy_update_interval == 0:
            loss_actor = (- self.critic1(torch.cat([s, self.get_policy(s)], -1))).mean()
            self.optimizer_actor.zero_grad()
            loss_actor.backward()
            self.optimizer_actor.step()
            self.update_target_net()

        self.optimizer_critic.zero_grad()
        loss_critic.backward()
        self.optimizer_critic.step()

        self.critic_update_cnt += 1

'''
TD3 test
'''
if __name__ == "__main__":
    env = gym.make("Pendulum-v0")
    device = 'cuda' if torch.cuda.is_available() else 'cpu'
    model = TD3(args = (3, 2, True, 2, -2, 30000, 0.01, device, 1e-3, 1e-3)).to(device)
    total_step = 0
    epsilon = 0.3
    for i in range(10000):
        s = env.reset()
        score = 0.
        epsilon = max(0.01, epsilon * 0.999)
        for t in range(200):
            total_step += 1
            action = model.select_action(torch.FloatTensor(s).to(device), epsilon)

            s_next, reward, done, info = env.step(action)
            model.save_trans((s, action, reward, s_next, done))
            score += reward
            s = s_next
            if len(model.replay_buffer.buffer) > 60:
                model.train()
            if done:
                break
        print("Epoch:{}    Score:{}    epsilon:{:.3}".format(i+1, score, epsilon))


import argparse
import glob
import os
import numpy as np
import pandas as pd
from Bio import SeqIO
from data import sequence_complement, INDEX_COLS, LFC_COLS, SEQUENCE_FEATS, SCALAR_FEATS

# relevant columns
GUIDE_COLS = ['end', 'start', 'junction_dist_5p', 'junction_dist_3p', 'guide_type', 'guide_seq']
TRANSCRIPT_COLS = ['target_seq', 'length', 'cds_start', 'cds_stop']
JUNCTION_COLS = ['junction_id', 'junction_seq', 'junction_category']
COLUMNS_TO_SAVE = INDEX_COLS + ['guide_type'] + LFC_COLS + SEQUENCE_FEATS + SCALAR_FEATS

def process_raw_wessels_data(sub_dir, num_folds, seed):
    """
    Process raw data from either Wessels et al. 2020 or 2022 and save the processed data as a Pandas DataFrame
    :param sub_dir: a subdirectory containing raw data from one of the Wessels et al. papers
    :param num_folds: number of validation folds
    :param seed: random number seed
    :return: None
    """
    # prepare guide file
    df_guide = pd.read_csv(os.path.join('data-raw', sub_dir, 'guide_table.txt'), delimiter='\t')
    df_guide.rename(columns={
        'TargetGene': 'gene',
        'GuideID': 'guide_id',
        'GuideSeq': 'guide_seq',
        'Type': 'guide_type',
        'Dist2Junction_5p': 'junction_dist_5p',
        'Dist2Junction_3p': 'junction_dist_3p'
    }, inplace=True)
    df_guide = df_guide[df_guide.gene != 'NT']
    df_guide = df_guide[df_guide.guide_type != 'NT']
    df_guide['guide_seq'] = df_guide['guide_seq'].apply(lambda seq: seq[::-1])
    for int_col in ['end', 'start', 'junction_dist_5p', 'junction_dist_3p']:
        df_guide[int_col] = df_guide[int_col].astype(int)
    df_guide = df_guide[INDEX_COLS + GUIDE_COLS]
    df_guide.set_index(INDEX_COLS, inplace=True)

    # prepare LFC file
    df_lfc = pd.DataFrame()
    for lfc_csv in glob.glob(os.path.join('data-raw', sub_dir, 'lfc', '*.csv')):
        df_lfc = pd.concat([df_lfc, pd.read_csv(lfc_csv)])
    df_lfc.rename(columns={'D30_R1': 'lfc_r1', 'D30_R2': 'lfc_r2', 'D30_R3': 'lfc_r3'}, inplace=True)
    df_lfc['gene'] = [s.split('_')[0] for s in df_lfc.index.values]
    df_lfc['guide_id'] = ['_'.join(s.split('_')[1:]) for s in df_lfc.index.values]
    df_lfc.reset_index(drop=True, inplace=True)
    df_lfc = df_lfc[INDEX_COLS + LFC_COLS]
    df_lfc.set_index(INDEX_COLS, inplace=True)

    # save non-targeting data
    try:
        df_nt = df_lfc.loc['NT', :].reset_index()[LFC_COLS]
        df_nt.to_pickle(os.path.join('data-processed', sub_dir + '-nt.bz2'))
    except KeyError:
        print(sub_dir, 'has no non-targeting guides!')

    # prepare transcript file
    df_transcripts = pd.read_csv(os.path.join('data-raw', sub_dir, 'transcript_lut.tsv'), delimiter='\t')
    df_transcripts.rename(columns={'gene_name': 'gene'}, inplace=True)
    df_transcripts['cds_start'] = df_transcripts['cds_coord'].apply(lambda s: s.split('-')[0]).astype(int)
    df_transcripts['cds_stop'] = df_transcripts['cds_coord'].apply(lambda s: s.split('-')[1]).astype(int)
    df_transcripts = load_transcripts(df_transcripts, os.path.join('data-raw', sub_dir, 'transcripts'), '.fasta')
    df_transcripts = df_transcripts[['gene'] + TRANSCRIPT_COLS]
    df_transcripts.set_index('gene', inplace=True)

    # load and join folding energy features
    df_fold_mfe = pd.read_csv(os.path.join('data-raw', sub_dir, 'features', 'foldMFE.txt'), delimiter='\t')
    del df_fold_mfe['Guide_Name']
    df_fold_mfe.rename(columns={
        'TargetGene': 'gene',
        'GuideID': 'guide_id',
        'MFE': 'mfe',
        'DR': 'direct_repeat',
        'Gquad': 'g_quad',
        'Fold': 'guide_ss',
    }, inplace=True)
    df_fold_mfe.set_index(INDEX_COLS, inplace=True)

    # load and join hybridization energy features
    df_hyb_mfe = pd.read_csv(os.path.join('data-raw', sub_dir, 'features', 'hybMFE.txt'), delimiter='\t')
    del df_hyb_mfe['Guide_Name']
    df_hyb_mfe.rename(columns={
        'TargetGene': 'gene',
        'GuideID': 'guide_id',
        'hybMFE_1.23': 'hybrid_mfe_1_23',
        'hybMFE_15.9': 'hybrid_mfe_15_9',
        'hybMFE_3.12': 'hybrid_mfe_3_12',
    }, inplace=True)
    df_hyb_mfe.set_index(INDEX_COLS, inplace=True)

    # target site accessibility features
    df_target_access = pd.read_csv(os.path.join('data-raw', sub_dir, 'features', 'targetAccess.txt'), delimiter='\t')
    del df_target_access['Guide_Name']
    df_target_access.rename(columns={
        'TargetGene': 'gene',
        'GuideID': 'guide_id',
        'Log10_Unpaired': 'log_unpaired',
        'Log10_Unpaired_p11': 'log_unpaired_11',
        'Log10_Unpaired_p19': 'log_unpaired_19',
        'Log10_Unpaired_p25': 'log_unpaired_25',
    }, inplace=True)
    df_target_access.set_index(['gene', 'guide_id'], inplace=True)

    # merge tables
    assert not df_guide.index.has_duplicates
    assert not df_lfc.index.has_duplicates
    assert not df_transcripts.index.has_duplicates
    df_scalar_features = df_fold_mfe.join(df_hyb_mfe, how='inner')
    df_scalar_features = df_scalar_features.join(df_target_access, how='inner')

    # aggregate the data
    df_data = df_guide.join(df_lfc, how='inner')
    df_data = df_data.join(df_transcripts, how='left')
    df_data = df_data.join(df_scalar_features, how='inner')

    # cut away non-targeted gene sequence, while keeping nearby sequence context
    for index, row in df_data.iterrows():
        df_data.loc[index, 'target_seq'] = row['target_seq'][row['end'] - 1:row['start']]
        df_data.loc[index, '5p_context'] = row['target_seq'][:row['end'] - 1]
        df_data.loc[index, '3p_context'] = row['target_seq'][row['start']:]

    # add normalized location of target site along transcript
    guide_loc = df_data[['start', 'end']].mean(axis=1)
    df_data['loc_utr_5p'] = guide_loc / (df_data['cds_start'] - 1)
    df_data['loc_cds'] = (guide_loc - df_data['cds_start']) / (df_data['cds_stop'] - df_data['cds_start'])
    df_data['loc_utr_3p'] = (guide_loc - df_data['cds_stop'] + 1) / (df_data['length'] - df_data['cds_stop'] + 1)
    for location_float in ['loc_utr_5p', 'loc_cds', 'loc_utr_3p']:
        df_data[location_float] = np.clip(df_data[location_float], a_min=0.0, a_max=1.0)
    df_data['log_gene_len'] = np.log10(df_data['length'])

    # keep only the columns of interest
    df_data.reset_index(inplace=True)
    keep_columns = [col for col in COLUMNS_TO_SAVE if col in df_data.columns]
    missing_columns = list(set(COLUMNS_TO_SAVE) - set(keep_columns))
    missing_columns.sort()
    print(sub_dir, 'data is missing:', missing_columns)
    df_data = df_data[keep_columns]

    # add fold assignments and save data
    df_data = fold_assignments(df_data, num_folds, seed)
    df_data.to_pickle(os.path.join('data-processed', sub_dir + '.bz2'))

def process_hap_titration_data(num_folds, seed):

    # load the data
    data_dir = os.path.join('data-raw', 'hap-titration')
    data = pd.read_csv(os.path.join(data_dir, 'Titration.GuideInfo.txt'), sep='\t').set_index('UID')
    lfc = pd.read_csv(os.path.join(data_dir, 'Titration.L2FCs.txt'), sep='\t')
    data = data.join(lfc[['HAP1_D14_R1', 'HAP1_D14_R2', 'HAP1_D14_R3']]).reset_index()

    # rename columns
    data = data.rename(columns={
        'GeneName': 'gene',
        'UID': 'guide_id',
        'Type': 'guide_type',
        'HAP1_D14_R1': 'lfc_r1',
        'HAP1_D14_R2': 'lfc_r2',
        'HAP1_D14_R3': 'lfc_r3',
        'Dist2Junction_5p': 'junction_dist_5p',
        'Dist2Junction_3p': 'junction_dist_3p',
        'hybrid_mfe_1_23': 'hybrid_mfe_1_23',
        'hybMFE_15.9': 'hybrid_mfe_15_9',
        'hybMFE_3.12': 'hybrid_mfe_3_12',
        'log_unpaired': 'log_unpaired',
        'log10_unpaired_p11': 'log_unpaired_11',
        'log10_unpaired_p19': 'log_unpaired_19',
        'log10_unpaired_p25': 'log_unpaired_25',
    })

    # save non-targeting data and remove it
    data_nt = data[data.guide_type == 'NT'].reset_index()[LFC_COLS]
    data_nt.to_pickle(os.path.join('data-processed', 'hap-titration-nt.bz2'))
    data = data[data.guide_type != 'NT']

    # sequence features
    data['guide_seq'] = data['GuideSeq'].apply(lambda seq: seq[::-1])
    data['5p_context'] = data['TargetSeqContext'].apply(lambda seq: seq[:2])
    data['target_seq'] = data['TargetSeqContext'].apply(lambda seq: seq[2:-2])
    data['3p_context'] = data['TargetSeqContext'].apply(lambda seq: seq[-2:])

    # target location
    data['loc_utr_5p'] = data['loc'].apply(lambda loc: loc[1:-1].split(';')[0])
    data['loc_cds'] = data['loc'].apply(lambda loc: loc[1:-1].split(';')[1])
    data['loc_utr_3p'] = data['loc'].apply(lambda loc: loc[1:-1].split(';')[2])

    # keep only the columns of interest
    data.reset_index(inplace=True)
    keep_columns = [col for col in COLUMNS_TO_SAVE if col in data.columns]
    missing_columns = list(set(COLUMNS_TO_SAVE) - set(keep_columns))
    missing_columns.sort()
    print('HAP titration data is missing:', missing_columns)
    data = data[keep_columns]

    # add fold assignments and save data
    data = fold_assignments(data, num_folds, seed)
    data.to_pickle(os.path.join('data-processed', 'hap-titration.bz2'))

def process_all_junctions():

    # features for all junctions
    data = pd.read_csv('data-raw/junction/230717_all_gencode_guides_hyb_table_unique.txt', delimiter='\t')
    data = data.rename(columns={
        'gene_name': 'gene',
        'ID': 'guide_id',
        'GuideSeq': 'guide_seq',
        'TargetSeqContext': 'target_seq',
        'hybMFE_15.9': 'hybrid_mfe_15_9',
        'hybMFE_3.12': 'hybrid_mfe_3_12',
        'log10_unpaired_p11': 'log_unpaired_11',
        'log10_unpaired_p19': 'log_unpaired_19',
        'log10_unpaired_p25': 'log_unpaired_25',
    })
    data['guide_type'] = 'PM'
    data['guide_seq'] = data['guide_seq'].apply(lambda seq: seq[::-1])
    data['5p_context'] = data['target_seq'].apply(lambda s: s[:2])
    data['3p_context'] = data['target_seq'].apply(lambda s: s[-2:])
    data['target_seq'] = data['target_seq'].apply(lambda s: s[2:-2])
    assert set(data['guide_seq'].apply(len).unique()) == set(data['target_seq'].apply(len).unique())
    data['log_gene_len'] = np.log10(data['txEnd'] - data['txStart'])
    data['strand'] = data['strand'].apply(lambda s: +1 if s == '+' else -1)

    # finalize data
    data = data[[col for col in COLUMNS_TO_SAVE if col in data.columns]]
    data.to_pickle(os.path.join('data-processed', 'junction-all.bz2'))

def process_qpcr_junction_data():

    # add qPCR data
    pcr = pd.read_csv('data-raw/junction/230726_all_qPCR.txt', sep='\t').set_index(['guide_id'])
    ids = pd.read_csv('data-raw/junction/230814_gencode_v41_universal_guideIDs.txt', sep='\t')
    assert len(ids) == ids['guide_sequence'].nunique()
    pcr = pd.merge(pcr, ids, how='left', on='guide_id')
    pcr = pcr.rename(columns={'guide_sequence': 'guide_seq', 'avg_qpcr': 'observed_lfc'})[['guide_seq', 'observed_lfc']]
    pcr['guide_seq'] = pcr['guide_seq'].apply(lambda seq: seq[::-1])

    # join additional features
    all_junctions = pd.read_pickle(os.path.join('data-processed', 'junction-all.bz2'))
    pcr = pd.merge(pcr, all_junctions, how='left', on='guide_seq')
    pcr.to_pickle(os.path.join('data-processed', 'junction-qpcr.bz2'))

def process_raw_junction_data(num_folds, seed):
    """
    Process raw junction data and save the processed data as a Pandas DataFrame
    :param num_folds: number of validation folds
    :param seed: random number seed
    :return: None
    """
    # base directory
    base_dir = os.path.join('data-raw', 'junction')

    # process non-targeting data
    data_nt = os.path.join(base_dir, '220301_R1_remove_TechPool_wno_batch_LFC_plus_a375_gene_expression.txt.gz')
    data_nt = pd.read_csv(data_nt, sep='\t', low_memory=False).rename(columns={'sgrna': 'guide_id'})
    data_nt = data_nt[data_nt.day == 'D21']
    data_nt = data_nt.pivot(index=['guide_id', 'type'], columns='replicate', values='logFC').reset_index('type')
    data_nt = data_nt.rename(columns={'R2': 'lfc_r2', 'R3': 'lfc_r3'})
    data_nt['lfc_r1'] = np.nan
    data_nt = data_nt.loc[data_nt.type == 'NT', LFC_COLS]
    data_nt.to_pickle(os.path.join('data-processed', 'junction-nt.bz2'))

    # load table containing LFC measurements
    data = os.path.join(base_dir, '230411_final_screen_table_essential_genes.txt')
    data = pd.read_csv(data, delimiter='\t', low_memory=False)
    data = data[data.day == 'D21']
    assert set(data['type'].unique()) == {'essential'}

    # adjust and rename the features we want to keep from this table
    data['guide_seq'] = data['guide_sequence'].apply(lambda seq: seq[::-1])
    data['junction_id'] = data['screen_junc_id']
    data['junction_olap_5p'] = (data['guide_start'] - data['junc_start']) / data['guide_sequence'].apply(len)
    data['junction_olap_3p'] = (data['guide_end'] - data['junc_end']) / data['guide_sequence'].apply(len)
    data['perc_gene_nuc'] = data['perc.gene.nuc'].apply(lambda x: x / 100)
    data['perc_junc_nuc'] = data['perc.junc.nuc'].apply(lambda x: x / 100)

    # pivot and rejoin replicate LFC values
    lfc = data[['guide_seq', 'replicate', 'logFC']].set_index('guide_seq')
    lfc = lfc.pivot(columns='replicate', values='logFC').rename(columns={'R2': 'lfc_r2', 'R3': 'lfc_r3'})
    data['lfc_r1'] = np.nan
    assert not lfc.index.has_duplicates
    data = data.set_index('guide_seq')
    data = data.loc[data.index.duplicated(keep='first')]
    data = data.join(lfc)
    assert not data.index.has_duplicates

    # join additional features
    all_junctions = pd.read_pickle(os.path.join('data-processed', 'junction-all.bz2'))
    all_junctions.set_index('guide_seq', inplace=True)
    assert not all_junctions.index.has_duplicates
    data = data[[c for c in data.columns if c in set(COLUMNS_TO_SAVE + JUNCTION_COLS) - set(all_junctions.columns)]]
    data = data.join(all_junctions).reset_index()

    # drop any rows containing NaNs
    data = data.loc[~data[list(set(data.columns) - {'lfc_r1'})].isna().any(axis=1)]

    # add junction sequence
    junction_sequence = pd.read_csv(os.path.join(base_dir, 'junc_seq.txt'), delimiter='\t', low_memory=False)
    junction_sequence.rename(columns={'junc.name': 'junction_id', 'junc.sequence': 'junction_seq'}, inplace=True)
    data = data.set_index('junction_id').join(junction_sequence.set_index('junction_id')['junction_seq']).reset_index()
    bad_targets = []
    for index, row in data.iterrows():
        target_seq = row['5p_context'] + row['target_seq'] + row['3p_context']
        if target_seq not in row['junction_seq']:
            bad_targets += [row['junction_id'] + ': ' + target_seq]
    print('Junctions where target + context sequence is not in junction sequence:')
    print('\n'.join(bad_targets))

    # report missing columns
    keep_columns = [col for col in COLUMNS_TO_SAVE if col in data.columns] + JUNCTION_COLS
    missing_columns = list(set(COLUMNS_TO_SAVE) - set(keep_columns))
    missing_columns.sort()
    print('Junction data is missing:', missing_columns)

    # add fold assignments and save data
    data = fold_assignments(data, num_folds, seed)
    data.to_pickle(os.path.join('data-processed', 'junction.bz2'))

def process_splice_site_data(num_folds, seed):
    # load junction guide data
    data = pd.read_pickle(os.path.join('data-processed', 'junction.bz2'))

    # reduce to splice sites
    data_splice_site = pd.DataFrame(data[['junction_seq'] + LFC_COLS].groupby('junction_seq')[LFC_COLS].mean())
    data_splice_site = data_splice_site.join(data[list(set(data.columns) - set(LFC_COLS))].set_index('junction_seq'))
    data_splice_site = data_splice_site.reset_index().drop_duplicates('junction_seq')
    data_splice_site['target_seq'] = data_splice_site['junction_seq']
    data_splice_site['5p_context'] = ''
    data_splice_site['3p_context'] = ''
    data_splice_site['guide_seq'] = data_splice_site['junction_seq'].apply(sequence_complement)
    data_splice_site['guide_id'] = data_splice_site['guide_seq']

    # keep only relevant columns
    keep_columns = [col for col in COLUMNS_TO_SAVE if col in data_splice_site.columns and col not in SCALAR_FEATS]
    data_splice_site = data_splice_site[keep_columns]

    # add fold assignments and save data
    data_splice_site = fold_assignments(data_splice_site, num_folds, seed)
    data_splice_site.to_pickle(os.path.join('data-processed', 'junction-splice-sites.bz2'))

def process_raw_junction_rbp_data():
    rna_prot_dir = os.path.join('data-raw/junction/RNAprot')

    # RNA prot junction-level predictions
    df_rbp_junc = pd.read_csv(os.path.join(rna_prot_dir, 'output_averaged.csv'))
    df_rbp_junc = df_rbp_junc.rename(columns=dict(site_id='junction_id')).set_index('junction_id')
    df_rbp_junc.to_pickle(os.path.join('data-processed', 'junction-rbp-junc.bz2'))

    # RNA prot junction-level predictions
    for (file, suffix) in zip(['all_peak_outputs.csv', 'relaxed_peak_outputs.csv'], ['nt', 'nt_relaxed']):
        df_rbp_nt = pd.read_csv(os.path.join(rna_prot_dir, file))
        df_rbp_nt = df_rbp_nt.rename(columns=dict(ref_id='junction_id'))
        if file == 'all_peak_outputs.csv':
            df_rbp_nt['RBP'] = df_rbp_nt['Cell_line'] + '_' + df_rbp_nt['Gene']
        elif file == 'relaxed_peak_outputs.csv':
            df_rbp_nt.rename(columns=dict(rbp='RBP'), inplace=True)
        else:
            raise NotImplementedError
        df_peak_s = df_rbp_nt.pivot_table(index='junction_id', columns='RBP', values='peak_region_s', fill_value=101)
        df_peak_e = df_rbp_nt.pivot_table(index='junction_id', columns='RBP', values='peak_region_e', fill_value=100)
        peaks = np.zeros(df_peak_s.shape + (101,), dtype=np.int8)
        np.put_along_axis(arr=peaks, indices=df_peak_s.values[..., None].astype(int) - 1, values=1, axis=-1)
        np.put_along_axis(arr=peaks, indices=df_peak_e.values[..., None].astype(int), values=-1, axis=-1)
        peaks = np.cumsum(peaks, axis=-1)[..., :100]
        df_rbp_nt = pd.DataFrame(index=df_peak_s.index, columns=df_peak_s.columns, data=peaks.tolist())
        df_rbp_nt.to_pickle(os.path.join('data-processed', 'junction-rbp-' + suffix + '.bz2'))

def process_raw_hap_validation_data():
    """
    Process raw junction isoform data and save the processed data as a Pandas DataFrame
    :return: None
    """
    # hap validation guides
    data_file = os.path.join('data-raw', 'hap-validation', 'validation_guides.csv')
    df_data = pd.read_csv(data_file)
    df_data.rename(columns={'top_sequence': 'guide_seq'}, inplace=True)

    # prepare guide/target sequence
    df_data['guide_seq'] = df_data['guide_seq'].apply(lambda seq: seq[::-1])
    df_data['target_seq'] = df_data['guide_seq'].apply(sequence_complement)

    # stuff missing values
    df_data['gene'] = 'unknown'
    df_data['guide_type'] = 'PM'
    df_data[['5p_context', '3p_context']] = ''

    # finalize data
    keep_columns = [col for col in COLUMNS_TO_SAVE if col in df_data.columns]
    missing_columns = list(set(COLUMNS_TO_SAVE) - set(keep_columns))
    missing_columns.sort()
    print('hap=validation', 'data is missing:', missing_columns)
    df_data = df_data[keep_columns]

    # save data
    df_data.to_pickle(os.path.join('data-processed', 'hap-validation.bz2'))

def load_transcripts(df_transcripts, transcript_dir, file_type):
    """
    Load transcript sequence for each gene in the provided transcript table
    :param df_transcripts: DataFrame with transcript IDs that point to transcript files
    :param transcript_dir: directory containing transcript files
    :param file_type: FASTA file type postfix (e.g. .fasta or .fa)
    :return: df_transcripts with gene sequences loaded into additional column
    """
    df_transcripts['target_seq'] = None
    for index, row in df_transcripts.iterrows():
        with open(os.path.join(transcript_dir, row['transcript_id'] + file_type), 'r') as file:
            seq_records = [s for s in SeqIO.parse(file, 'fasta')]
            assert len(seq_records) == 1
            target_seq = str(seq_records[0].seq)
            assert len(target_seq) == row['length']
            df_transcripts.loc[index, 'target_seq'] = target_seq
    return df_transcripts

def fold_assignments(df_data, num_folds, seed):
    # set random number seed
    np.random.seed(seed)

    # guide folds
    df_data['guide_fold'] = 1 + np.random.choice(num_folds, len(df_data))

    # target folds
    df_pm = df_data[df_data.guide_type == 'PM'][['target_seq']]
    df_pm['target_fold'] = 1 + np.random.choice(num_folds, len(df_pm))
    df_data = pd.merge(df_data, df_pm, how='inner', on='target_seq')

    return df_data

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--dataset', type=str, default='all', help='which dataset to process')
    parser.add_argument('--num_folds', type=int, default=10, help='number of folds')
    parser.add_argument('--seed', type=int, default=112358, help='random number seed for fold assignments')
    args = parser.parse_args()

    # make sure output directory exists
    os.makedirs('data-processed', exist_ok=True)

    # prepare requested data sources
    if args.dataset == 'all' or args.dataset == 'flow-cytometry':
        process_raw_wessels_data('flow-cytometry', args.num_folds, args.seed)
    if args.dataset == 'all' or args.dataset == 'off-target':
        process_raw_wessels_data('off-target', args.num_folds, args.seed)
    if args.dataset == 'all' or args.dataset == 'hap-titration':
        process_hap_titration_data(args.num_folds, args.seed)
    if args.dataset == 'all' or args.dataset == 'junction':
        process_all_junctions()
        process_qpcr_junction_data()
        process_raw_junction_data(args.num_folds, args.seed)
        process_splice_site_data(args.num_folds, args.seed)
        # process_raw_junction_rbp_data()
    # if args.dataset == 'all' or args.dataset == 'hap-validation':
    #     process_raw_hap_validation_data()


from monitor.models import MonitorHost,Trigger
import json
import time
class StatusSerializer(object):
    def __init__(self,request,redis):
        self.request = request
        self.redis = redis

    def by_hosts(self):
        '''
        serialize all the hosts
        :return:
        '''
        host_obj_list = MonitorHost.objects.all()
        host_data_list = []
        for h in host_obj_list:
            host_data_list.append( self.single_host_info(h)  )
        return host_data_list

    def single_host_info(self,host_obj):
        '''
        serialize single host into a dic
        :param host_obj:
        :return:
        '''
        data = {
            'id': host_obj.id,
            'name':host_obj.asset.objects.first().server.hostname,
            'ip_addr':host_obj.asset.objects.first().server.hostname,
            'status': host_obj.asset.device_status_id,
            'uptime': None,
            'last_update':None,
            'total_services':None,
            'ok_nums':None,

        }

        #for uptime
        uptime = self.get_host_uptime(host_obj)
        self.get_triggers(host_obj)
        if uptime:
            # print('uptime:',uptime)
            data['uptime'] = uptime[0]['uptime']
            print('mktime :',time.gmtime(uptime[1]))
            data['last_update'] = time.strftime("%Y-%m-%d %H:%M:%S",time.localtime(uptime[1]))

        #for triggers
        data['triggers'] = self.get_triggers(host_obj)

        return  data

    def get_host_uptime(self,host_obj):
        '''
        get host uptime data
        :param host_obj:
        :return:
        '''

        redis_key = 'StatusData_%s_uptime_latest' % host_obj.id
        last_data_point = self.redis.lrange(redis_key,-1,-1)
        if last_data_point:
            #print('----last updtime point:',last_data_point[0])
            last_data_point,last_update = json.loads(last_data_point[0])
            return last_data_point,last_update

    def get_triggers(self,host_obj):
        trigger_keys = self.redis.keys("host_%s_trigger_*" % host_obj.id)
        #print('trigger keys:',trigger_keys)
        ''' (1,'Information'),
        (2,'Warning'),
        (3,'Average'),
        (4,'High'),
        (5,'Diaster'), '''
        trigger_dic = {
            1 : [],
            2 : [],
            3 : [],
            4 : [],
            5 : []
        }

        for trigger_key in trigger_keys:
            trigger_data = self.redis.get(trigger_key)
            print("trigger_key",trigger_key)
            if trigger_key.decode().endswith("None"):
                trigger_dic[4].append(json.loads(trigger_data.decode()))
            else:
                trigger_id = trigger_key.decode().split('_')[-1]
                trigger_obj = Trigger.objects.get(id=trigger_id)
                trigger_dic[trigger_obj.severity].append(json.loads(trigger_data.decode()))

        #print('triiger data',trigger_dic)
        return trigger_dic


import siibra
from nilearn import plotting
import numpy as np
import matplotlib.pyplot as plt
# sphinx_gallery_thumbnail_path = '_static/example_thumbnails/02-006.png'

# %%
# Request the BigBrain cortical layer parcellation by Wagstyl et al.
# from siibra.
layermap = siibra.get_map(parcellation='layers', space="big brain")
layermap.regions

# %%
# We fetch the surface mesh of a layer by specifying a unique part of its name.
l4_surf_l = layermap.fetch(region="layer 4 left", format="mesh")

# %%
# We can also choose individual hemispheres, and recombine meshes.
# For illustration, we create a combined mesh of the white matter surface
# in the left hemisphere and the layer 1 surface of the right hemisphere.
wm_surf_r = layermap.fetch(region="non-cortical right", format="mesh")
mesh = siibra.commons.merge_meshes([l4_surf_l, wm_surf_r], labels=[10, 20])

# %%
plotting.view_surf(
    (mesh['verts'], mesh['faces']),
    surf_map=mesh['labels'],
    cmap='Set1', vmin=10, vmax=30, symmetric_cmap=False, colorbar=False
)

# %%
# Since the cortical layer surfaces share corresponding vertices,
# we can compute approximate layer thicknesses as Euclidean distances
# of mesh vertices. Here we compare the layer depth distribution of
# layers 4 and 5 across the left hemisphere.
plt.figure()
layers = [2, 4, 5]
thicknesses = {}
for layer in layers:
    upper_surf = layermap.fetch(f"layer {layer+1} left", format="mesh")
    lower_surf = layermap.fetch(f"layer {layer} left", format="mesh")
    thicknesses[layer] = np.linalg.norm(upper_surf['verts'] - lower_surf['verts'], axis=1)
    plt.hist(thicknesses[layer], 200)
plt.xlabel('Layer depth (left hemisphere)')
plt.ylabel('# vertices')
plt.legend([f"Layer {_}" for _ in layers])
plt.grid(True)

# %%
# We can plot the (last) thickness distribution on the surface
plotting.view_surf(
    (lower_surf['verts'], lower_surf['faces']),
    surf_map=thicknesses[4],
    symmetric_cmap=False, cmap='magma', vmax=0.3
)
# %%


import wx

app = wx.App(False)
SCREEN_WIDTH, SCREEN_HEIGHT = wx.GetDisplaySize()

SCREEN_WIDTH*=1.5
SCREEN_HEIGHT*=1.5

row, col = (SCREEN_HEIGHT*0.1, SCREEN_WIDTH*0.1)
col1, col2, col3, col4, col5, col6, col7, col8, col9, col10 = [col * x for x in range(1, 11)]
row1, row2, row3, row4, row5, row6, row7, row8, row9, row10 = [row * x for x in range(1, 11)]


from rest_framework import serializers

from core.models import Product, Investor, Purchase

class ProductSerializer(serializers.ModelSerializer):
    class Meta:
        model = Product
        fields = '__all__'

class InvestorSerializer(serializers.ModelSerializer):
    class Meta:
        model = Investor
        fields = '__all__'

class PurchaseSerializer(serializers.ModelSerializer):
    percentage = serializers.SerializerMethodField()
    class Meta:
        model = Purchase
        # fields = '__all__'
        fields = ["id", "percentage", "amount_to_sell", "left_amount", "investor", "product"]

    def get_percentage(self, obj):
        return obj.percentage