import re
from typing import Any, Callable, Dict, List, Optional, Union

import pandas as pd

from snowflake.ml._internal import telemetry
from snowflake.ml._internal.utils import sql_identifier
from snowflake.ml.model import model_signature
from snowflake.ml.model._client.model import model_method_info
from snowflake.ml.model._client.ops import metadata_ops, model_ops
from snowflake.snowpark import dataframe

_TELEMETRY_PROJECT = "MLOps"
_TELEMETRY_SUBPROJECT = "ModelManagement"

class ModelVersion:
    """Model Version Object representing a specific version of the model that could be run."""

    _model_ops: model_ops.ModelOperator
    _model_name: sql_identifier.SqlIdentifier
    _version_name: sql_identifier.SqlIdentifier

    def __init__(self) -> None:
        raise RuntimeError("ModelVersion's initializer is not meant to be used. Use `version` from model instead.")

    @classmethod
    def _ref(
        cls,
        model_ops: model_ops.ModelOperator,
        *,
        model_name: sql_identifier.SqlIdentifier,
        version_name: sql_identifier.SqlIdentifier,
    ) -> "ModelVersion":
        self: "ModelVersion" = object.__new__(cls)
        self._model_ops = model_ops
        self._model_name = model_name
        self._version_name = version_name
        return self

    def __eq__(self, __value: object) -> bool:
        if not isinstance(__value, ModelVersion):
            return False
        return (
            self._model_ops == __value._model_ops
            and self._model_name == __value._model_name
            and self._version_name == __value._version_name
        )

    @property
    def model_name(self) -> str:
        return self._model_name.identifier()

    @property
    def version_name(self) -> str:
        return self._version_name.identifier()

    @property
    def fully_qualified_model_name(self) -> str:
        return self._model_ops._model_version_client.fully_qualified_model_name(self._model_name)

    @property
    @telemetry.send_api_usage_telemetry(
        project=_TELEMETRY_PROJECT,
        subproject=_TELEMETRY_SUBPROJECT,
    )
    def description(self) -> str:
        statement_params = telemetry.get_statement_params(
            project=_TELEMETRY_PROJECT,
            subproject=_TELEMETRY_SUBPROJECT,
        )
        return self._model_ops.get_comment(
            model_name=self._model_name,
            version_name=self._version_name,
            statement_params=statement_params,
        )

    @description.setter
    @telemetry.send_api_usage_telemetry(
        project=_TELEMETRY_PROJECT,
        subproject=_TELEMETRY_SUBPROJECT,
    )
    def description(self, description: str) -> None:
        statement_params = telemetry.get_statement_params(
            project=_TELEMETRY_PROJECT,
            subproject=_TELEMETRY_SUBPROJECT,
        )
        return self._model_ops.set_comment(
            comment=description,
            model_name=self._model_name,
            version_name=self._version_name,
            statement_params=statement_params,
        )

    @telemetry.send_api_usage_telemetry(
        project=_TELEMETRY_PROJECT,
        subproject=_TELEMETRY_SUBPROJECT,
    )
    def list_metrics(self) -> Dict[str, Any]:
        """Show all metrics logged with the model version.

        Returns:
            A dictionary showing the metrics
        """
        statement_params = telemetry.get_statement_params(
            project=_TELEMETRY_PROJECT,
            subproject=_TELEMETRY_SUBPROJECT,
        )
        return self._model_ops._metadata_ops.load(
            model_name=self._model_name, version_name=self._version_name, statement_params=statement_params
        )["metrics"]

    @telemetry.send_api_usage_telemetry(
        project=_TELEMETRY_PROJECT,
        subproject=_TELEMETRY_SUBPROJECT,
    )
    def get_metric(self, metric_name: str) -> Any:
        """Get the value of a specific metric.

        Args:
            metric_name: The name of the metric

        Raises:
            KeyError: Raised when the requested metric name does not exist.

        Returns:
            The value of the metric.
        """
        metrics = self.list_metrics()
        if metric_name not in metrics:
            raise KeyError(f"Cannot find metric with name {metric_name}.")
        return metrics[metric_name]

    @telemetry.send_api_usage_telemetry(
        project=_TELEMETRY_PROJECT,
        subproject=_TELEMETRY_SUBPROJECT,
    )
    def set_metric(self, metric_name: str, value: Any) -> None:
        """Set the value of a specific metric name

        Args:
            metric_name: The name of the metric
            value: The value of the metric.
        """
        statement_params = telemetry.get_statement_params(
            project=_TELEMETRY_PROJECT,
            subproject=_TELEMETRY_SUBPROJECT,
        )
        metrics = self.list_metrics()
        metrics[metric_name] = value
        self._model_ops._metadata_ops.save(
            metadata_ops.ModelVersionMetadataSchema(metrics=metrics),
            model_name=self._model_name,
            version_name=self._version_name,
            statement_params=statement_params,
        )

    @telemetry.send_api_usage_telemetry(
        project=_TELEMETRY_PROJECT,
        subproject=_TELEMETRY_SUBPROJECT,
    )
    def delete_metric(self, metric_name: str) -> None:
        """Delete a metric from metric storage.

        Args:
            metric_name: The name of the metric to be deleted.

        Raises:
            KeyError: Raised when the requested metric name does not exist.
        """
        statement_params = telemetry.get_statement_params(
            project=_TELEMETRY_PROJECT,
            subproject=_TELEMETRY_SUBPROJECT,
        )
        metrics = self.list_metrics()
        if metric_name not in metrics:
            raise KeyError(f"Cannot find metric with name {metric_name}.")
        del metrics[metric_name]
        self._model_ops._metadata_ops.save(
            metadata_ops.ModelVersionMetadataSchema(metrics=metrics),
            model_name=self._model_name,
            version_name=self._version_name,
            statement_params=statement_params,
        )

    @telemetry.send_api_usage_telemetry(
        project=_TELEMETRY_PROJECT,
        subproject=_TELEMETRY_SUBPROJECT,
    )
    def list_methods(self) -> List[model_method_info.ModelMethodInfo]:
        """List all method information in a model version that is callable.

        Returns:
            A list of ModelMethodInfo object containing the following information:
            - name: The name of the method to be called (both in SQL and in Python SDK).
            - target_method: The original method name in the logged Python object.
            - Signature: Python signature of the original method.
        """
        statement_params = telemetry.get_statement_params(
            project=_TELEMETRY_PROJECT,
            subproject=_TELEMETRY_SUBPROJECT,
        )
        # TODO(SNOW-986673, SNOW-986675): Avoid parsing manifest and meta file and put Python signature into user_data.
        manifest = self._model_ops.get_model_version_manifest(
            model_name=self._model_name,
            version_name=self._version_name,
            statement_params=statement_params,
        )
        model_meta = self._model_ops.get_model_version_native_packing_meta(
            model_name=self._model_name,
            version_name=self._version_name,
            statement_params=statement_params,
        )
        return_methods_info: List[model_method_info.ModelMethodInfo] = []
        for method in manifest["methods"]:
            # Method's name is resolved so we need to use case_sensitive as True to get the user-facing identifier.
            method_name = sql_identifier.SqlIdentifier(method["name"], case_sensitive=True).identifier()
            # Method's handler is `functions.<target_method>.infer`
            assert re.match(
                r"^functions\.([^\d\W]\w*)\.infer$", method["handler"]
            ), f"Get unexpected handler name {method['handler']}"
            target_method = method["handler"].split(".")[1]
            signature_dict = model_meta["signatures"][target_method]
            method_info = model_method_info.ModelMethodInfo(
                name=method_name,
                target_method=target_method,
                signature=model_signature.ModelSignature.from_dict(signature_dict),
            )
            return_methods_info.append(method_info)

        return return_methods_info

    @telemetry.send_api_usage_telemetry(
        project=_TELEMETRY_PROJECT,
        subproject=_TELEMETRY_SUBPROJECT,
    )
    def run(
        self,
        X: Union[pd.DataFrame, dataframe.DataFrame],
        *,
        method_name: Optional[str] = None,
    ) -> Union[pd.DataFrame, dataframe.DataFrame]:
        """Invoke a method in a model version object

        Args:
            X: The input data. Could be pandas DataFrame or Snowpark DataFrame
            method_name: The method name to run. It is the name you will use to call a method in SQL. Defaults to None.
                It can only be None if there is only 1 method.

        Raises:
            ValueError: No method with the corresponding name is available.
            ValueError: There are more than 1 target methods available in the model but no method name specified.

        Returns:
            The prediction data.
        """
        statement_params = telemetry.get_statement_params(
            project=_TELEMETRY_PROJECT,
            subproject=_TELEMETRY_SUBPROJECT,
        )

        methods: List[model_method_info.ModelMethodInfo] = self.list_methods()
        if method_name:
            req_method_name = sql_identifier.SqlIdentifier(method_name).identifier()
            find_method: Callable[[model_method_info.ModelMethodInfo], bool] = (
                lambda method: method["name"] == req_method_name
            )
            target_method_info = next(
                filter(find_method, methods),
                None,
            )
            if target_method_info is None:
                raise ValueError(
                    f"There is no method with name {method_name} available in the model"
                    f" {self.fully_qualified_model_name} version {self.version_name}"
                )
        elif len(methods) != 1:
            raise ValueError(
                f"There are more than 1 target methods available in the model {self.fully_qualified_model_name}"
                f" version {self.version_name}. Please specify a `method_name` when calling the `run` method."
            )
        else:
            target_method_info = methods[0]
        return self._model_ops.invoke_method(
            method_name=sql_identifier.SqlIdentifier(target_method_info["name"]),
            signature=target_method_info["signature"],
            X=X,
            model_name=self._model_name,
            version_name=self._version_name,
            statement_params=statement_params,
        )


from typing import List

from fastapi import APIRouter, Depends, HTTPException, Response
from .. import models, schemas, database, oauth2
from sqlalchemy.orm import Session
from sqlalchemy.sql.expression import text

router = APIRouter(prefix="/tasks",
                   tags=['Tasks'])

@router.get('/', response_model=List[schemas.TaskOut])
def get_tasks(db: Session = Depends(database.get_db),
              user: schemas.UserOut = Depends(oauth2.get_current_user)):
    tasks = db.query(models.Tasks).filter(models.Tasks.owner_id == user.id).all()

    return tasks

@router.get('/{id}', response_model=schemas.TaskOut)
def get_task_by_id(id: int, db: Session = Depends(database.get_db),
                   user: schemas.UserOut = Depends(oauth2.get_current_user)):
    task = db.query(models.Tasks).filter(models.Tasks.id == id).first()
    if not task:
        raise HTTPException(status_code=404,
                            detail=f"Task with id {id} does not exist")
    if task.owner_id != user.id:
        raise HTTPException(status_code=403,
                            detail="Not authorized for this task")
    return task

@router.post('/', response_model=schemas.TaskOut)
def add_task(task: schemas.TaskAdd, db: Session = Depends(database.get_db),
             user: schemas.UserOut = Depends(oauth2.get_current_user)):
    new_task = models.Tasks(owner_id=user.id, **task.dict())
    db.add(new_task)
    db.commit()
    db.refresh(new_task)

    return new_task

@router.patch('/{id}', response_model=schemas.TaskOut)
def complete_task(id: int, db: Session = Depends(database.get_db),
                  user: schemas.UserOut = Depends(oauth2.get_current_user)):
    task_query = db.query(models.Tasks).filter(models.Tasks.id == id)
    task = task_query.first()
    if not task:
        raise HTTPException(status_code=404,
                            detail=f"Task with id {id} does not exist")
    if task.owner_id != user.id:
        raise HTTPException(status_code=403,
                            detail="Not authorized for this task")
    if task.completed:
        raise HTTPException(status_code=403
                            ,detail="Task is already complete")

    task_query.update({"completed": True,
                       "completed_at": text("NOW()")})
    db.commit()
    db.refresh(task)
    return task

@router.delete('/{id}')
def delete_task(id: int, db: Session = Depends(database.get_db),
                  user: schemas.UserOut = Depends(oauth2.get_current_user)):
    task_query = db.query(models.Tasks)
    task = task_query.filter(models.Tasks.id == id).first()
    if not task:
        raise HTTPException(status_code=404,
                            detail=f"Task with id {id} does not exist")
    if task.owner_id != user.id:
        raise HTTPException(status_code=403,
                            detail="Not authorized for this task")

    db.delete(task)
    db.commit()

    return Response(status_code=204)


import sys
from platform import system
from os import makedirs
from os.path import isdir, join

IS_WINDOWS = sys.platform.startswith("win")
IS_MACOS = sys.platform.startswith("darwin")

from SCons.Script import (ARGUMENTS, COMMAND_LINE_TARGETS, AlwaysBuild,
                          Builder, Default, DefaultEnvironment)

from platformio.public import list_serial_ports

def BeforeUpload(target, source, env):  # pylint: disable=W0613,W0621
    env.AutodetectUploadPort()

    upload_options = {}
    if "BOARD" in env:
        upload_options = env.BoardConfig().get("upload", {})

    if not bool(upload_options.get("disable_flushing", False)):
        env.FlushSerialBuffer("$UPLOAD_PORT")

    before_ports = list_serial_ports()

    if bool(upload_options.get("use_1200bps_touch", False)):
        env.TouchSerialPort("$UPLOAD_PORT", 1200)

    if bool(upload_options.get("wait_for_upload_port", False)):
        env.Replace(UPLOAD_PORT=env.WaitForNewSerialPort(before_ports))

def AfterUpload(target, source, env):  # pylint: disable=W0613,W0621
    before_ports = list_serial_ports()
    env.WaitForNewSerialPort(before_ports)

env = DefaultEnvironment()
platform = env.PioPlatform()
board = env.BoardConfig()
board_nrf_type = int(board.get("build.mcu", "")[3:5])

env.Replace(
    AR="arm-none-eabi-ar",
    AS="arm-none-eabi-as",
    CC="arm-none-eabi-gcc",
    CXX="arm-none-eabi-g++",
    GDB="arm-none-eabi-gdb",
    OBJCOPY="arm-none-eabi-objcopy",
    RANLIB="arm-none-eabi-ranlib",
    SIZETOOL="arm-none-eabi-size",

    ARFLAGS=["rc"],

    SIZEPROGREGEXP=r"^(?:\.text|\.data|\.rodata|\.text.align|\.ARM.exidx)\s+(\d+).*",
    SIZEDATAREGEXP=r"^(?:\.data|\.bss|\.noinit)\s+(\d+).*",
    SIZECHECKCMD="$SIZETOOL -A -d $SOURCES",
    SIZEPRINTCMD='$SIZETOOL -B -d $SOURCES',

    ERASEFLAGS=["--eraseall", "-f", "nrf%d" % board_nrf_type],
    ERASECMD="nrfjprog $ERASEFLAGS",

    PROGSUFFIX=".elf"
)

# Allow user to override via pre:script
if env.get("PROGNAME", "program") == "program":
    env.Replace(PROGNAME="firmware")

env.Append(
    BUILDERS=dict(
        ElfToBin=Builder(
            action=env.VerboseAction(" ".join([
                "$OBJCOPY",
                "-O",
                "binary",
                "$SOURCES",
                "$TARGET"
            ]), "Building $TARGET"),
            suffix=".bin"
        ),
        ElfToHex=Builder(
            action=env.VerboseAction(" ".join([
                "$OBJCOPY",
                "-O",
                "ihex",
                "-R",
                ".eeprom",
                "$SOURCES",
                "$TARGET"
            ]), "Building $TARGET"),
            suffix=".hex"
        ),
    )
)

upload_protocol = env.subst("$UPLOAD_PROTOCOL")

if upload_protocol not in board.get("upload.protocols"):
    sys.stderr.write("%s is not a valid upload option for %s" % (upload_protocol, board.get("name", "")))
    env.Exit(1)

if upload_protocol == "nrfutil":
    env.Append(
        BUILDERS=dict(
            PackageDfu=Builder(
                action=env.VerboseAction(" ".join([
                    "nrfutil",
                    "pkg",
                    "generate",
                    "--hw-version",
                    "%d" % board_nrf_type,
                    "--sd-req",
                    "0",
                    "--debug-mode",
                    "--application",
                    "$SOURCES",
                    "$TARGET"
                ]), "Building $TARGET"),
                suffix=".zip"
            ),
        )
    )

elif "adafruit-nrfutil" == upload_protocol:
    env.Append(
        BUILDERS=dict(
            PackageDfu=Builder(
                action=env.VerboseAction(" ".join([
                    '"$PYTHONEXE"',
                    '"%s"' % join(platform.get_package_dir(
                        "tool-adafruit-nrfutil") or "", "adafruit-nrfutil.py"),
                    "dfu",
                    "genpkg",
                    "--dev-type",
                    "0x0052",
                    "--application",
                    "$SOURCES",
                    "$TARGET"
                ]), "Building $TARGET"),
                suffix=".zip"
            ),
        )
    )

#
# Target: Build executable and linkable firmware
#

target_elf = None
if "nobuild" in COMMAND_LINE_TARGETS:
    target_elf = join("$BUILD_DIR", "${PROGNAME}.elf")
    target_firm = join("$BUILD_DIR", "${PROGNAME}.hex")
else:
    target_elf = env.BuildProgram()
    if "nrfutil" == upload_protocol:
        target_firm = env.PackageDfu(
            join("$BUILD_DIR", "${PROGNAME}"),
            env.ElfToHex(join("$BUILD_DIR", "${PROGNAME}"), target_elf))
    elif "adafruit-nrfutil" == upload_protocol:
        target_firm = env.PackageDfu(
            join("$BUILD_DIR", "${PROGNAME}"),
            env.ElfToHex(join("$BUILD_DIR", "${PROGNAME}"), target_elf))
    else:
        target_firm = env.ElfToHex(
            join("$BUILD_DIR", "${PROGNAME}"), target_elf)
    env.Depends(target_firm, "checkprogsize")

AlwaysBuild(env.Alias("nobuild", target_firm))
target_buildprog = env.Alias("buildprog", target_firm, target_firm)

#
# Target: Print binary size
#

target_size = env.AddPlatformTarget(
    "size",
    target_elf,
    env.VerboseAction("$SIZEPRINTCMD", "Calculating size $SOURCE"),
    "Program Size",
    "Calculate program size",
)

#
# Target: Upload by default .bin file
#

debug_tools = board.get("debug.tools", {})
upload_actions = []

if upload_protocol.startswith("blackmagic"):
    env.Replace(
        UPLOADER="$GDB",
        UPLOADERFLAGS=[
            "-nx",
            "--batch",
            "-ex", "target extended-remote %s$UPLOAD_PORT" % '\\\\.\\' if IS_WINDOWS else '',
            "-ex", "monitor %s_scan" %
            ("jtag" if upload_protocol == "blackmagic-jtag" else "swdp"),
            "-ex", "attach 1",
            "-ex", "load",
            "-ex", "compare-sections",
            "-ex", "kill"
        ],
        ERASEFLAGS=[
            "-nx",
            "--batch",
            "-ex", "target extended-remote %s$UPLOAD_PORT" % '\\\\.\\' if IS_WINDOWS else '',
            "-ex", "monitor %s_scan" %
            ("jtag" if upload_protocol == "blackmagic-jtag" else "swdp"),
            "-ex", "attach 1",
            "-ex", "monitor erase_mass",
            "-ex", "kill"
        ],
        UPLOADCMD="$UPLOADER $UPLOADERFLAGS $BUILD_DIR/${PROGNAME}.elf",
        ERASECMD="$UPLOADER $ERASEFLAGS"

    )
    upload_actions = [
        env.VerboseAction(env.AutodetectUploadPort, "Looking for BlackMagic port..."),
        env.VerboseAction("$UPLOADCMD", "Uploading $SOURCE")
    ]

elif upload_protocol == "nrfjprog":
    env.Replace(
        UPLOADER="nrfjprog",
        UPLOADERFLAGS=[
            "--chiperase",
            "--reset"
        ],
        UPLOADCMD="$UPLOADER $UPLOADERFLAGS --program $SOURCE"
    )
    upload_actions = [env.VerboseAction("$UPLOADCMD", "Uploading $SOURCE")]

elif upload_protocol == "nrfutil":
    try:
        import nordicsemi
    except ImportError:
        env.Execute('$PYTHONEXE -m pip install -U --force-reinstall nrfutil==6.1.4')

    env.Replace(
        UPLOADER="nrfutil",
        UPLOADERFLAGS=[
            "dfu",
            "serial",
            "-p",
            "$UPLOAD_PORT",
            "-b",
            "$UPLOAD_SPEED",
        ],
        UPLOADCMD='$UPLOADER $UPLOADERFLAGS -pkg $SOURCE'
    )
    upload_actions = [
        env.VerboseAction(BeforeUpload, "Looking for upload port..."),
        env.VerboseAction("$UPLOADCMD", "Uploading $SOURCE"),
        env.VerboseAction(AfterUpload, "Looking for upload port..."),
    ]

elif upload_protocol == "adafruit-nrfutil":
    env.Replace(
        UPLOADER=join(platform.get_package_dir(
            "tool-adafruit-nrfutil") or "", "adafruit-nrfutil.py"),
        UPLOADERFLAGS=[
            "dfu",
            "serial",
            "-p",
            "$UPLOAD_PORT",
            "-b",
            "$UPLOAD_SPEED",
            "--singlebank",
        ],
        UPLOADCMD='"$PYTHONEXE" "$UPLOADER" $UPLOADERFLAGS -pkg $SOURCE'
    )
    upload_actions = [
        env.VerboseAction(BeforeUpload, "Looking for upload port..."),
        env.VerboseAction("$UPLOADCMD", "Uploading $SOURCE")
    ]

elif upload_protocol.startswith("jlink"):
    def _jlink_cmd_script(env, source):
        build_dir = env.subst("$BUILD_DIR")
        if not isdir(build_dir):
            makedirs(build_dir)
        script_path = join(build_dir, "upload.jlink")
        commands = [
            "h",
            "loadbin %s, %s" % (source, board.get(
                "upload.offset_address", "0x0")),
            "r",
            "q"
        ]
        with open(script_path, "w") as fp:
            fp.write("\n".join(commands))
        return script_path

    def _jlink_erase_cmd_script(env):
        build_dir = env.subst("$BUILD_DIR")
        if not isdir(build_dir):
            makedirs(build_dir)
        script_path = join(build_dir, "erase.jlink")
        commands = [
            "erase",
            "q"
        ]
        with open(script_path, "w") as fp:
            fp.write("\n".join(commands))
        return script_path

    env.Replace(
        __jlink_cmd_script=_jlink_cmd_script,
        __jlink_erase_cmd_script=_jlink_erase_cmd_script,
        UPLOADER="JLink.exe" if system() == "Windows" else "JLinkExe",
        UPLOADERFLAGS=[
            "-device", board.get("debug", {}).get("jlink_device"),
            "-speed", env.GetProjectOption("debug_speed", "4000"),
            "-if", ("jtag" if upload_protocol == "jlink-jtag" else "swd"),
            "-autoconnect", "1",
            "-NoGui", "1"
        ],
        UPLOADCMD='$UPLOADER $UPLOADERFLAGS -CommanderScript "${__jlink_cmd_script(__env__, SOURCE)}"',
        ERASECMD='$UPLOADER $UPLOADERFLAGS -CommanderScript "${__jlink_erase_cmd_script(__env__)}"'
    )
    upload_actions = [env.VerboseAction("$UPLOADCMD", "Uploading $SOURCE")]

elif upload_protocol in debug_tools:
    openocd_args = [
        "-d%d" % (2 if int(ARGUMENTS.get("PIOVERBOSE", 0)) else 1)
    ]
    openocd_args.extend(
        debug_tools.get(upload_protocol).get("server").get("arguments", []))
    if env.GetProjectOption("debug_speed"):
        openocd_args.extend(
            ["-c", "adapter speed %s" % env.GetProjectOption("debug_speed")]
        )
    openocd_args.extend([
        "-c", "program {$SOURCE} %s verify reset; shutdown;" %
        board.get("upload.offset_address", "")
    ])
    openocd_args = [
        f.replace("$PACKAGE_DIR",
                  platform.get_package_dir("tool-openocd") or "")
        for f in openocd_args
    ]
    openocd_erase = openocd_args.copy()
    openocd_erase.pop()
    openocd_erase.extend(["init; halt; nrf5 mass_erase; reset; shutdown;"])
    env.Replace(
        UPLOADER="openocd",
        UPLOADERFLAGS=openocd_args,
        UPLOADCMD="$UPLOADER $UPLOADERFLAGS",
        ERASEFLAGS=openocd_erase,
        ERASECMD="$UPLOADER $ERASEFLAGS"
    )
    upload_actions = [env.VerboseAction("$UPLOADCMD", "Uploading $SOURCE")]

# custom upload tool
elif upload_protocol == "custom":
    upload_actions = [env.VerboseAction("$UPLOADCMD", "Uploading $SOURCE")]

else:
    sys.stderr.write("Warning! Unknown upload protocol %s\n" % upload_protocol)

env.AddPlatformTarget("upload", target_firm, upload_actions, "Upload")

#
# Target: Flash bootloader
#

if "BOOTLOADERHEX" in env:
    if "bootloader" in COMMAND_LINE_TARGETS and upload_protocol.startswith("blackmagic"):
        env.Replace(
            UPLOADCMD="$UPLOADER $UPLOADERFLAGS %s" % env.get("BOOTLOADERHEX", "")
        )

    env.AddPlatformTarget(
        "bootloader",
        env.get("BOOTLOADERHEX", ""),
        upload_actions,
        "Burn Bootloader",
    )

if "bootloader" in COMMAND_LINE_TARGETS and not env.GetProjectOption("board_bootloader", ""):
    sys.stderr.write("Error. board_bootloader not specified in environment.\n")
    env.Exit(1)

#
# Target: Erase Flash
#

env.AddPlatformTarget(
    "erase", None, env.VerboseAction("$ERASECMD", "Erasing..."), "Erase Flash")

#
# Information about obsolete method of specifying linker scripts
#

if any("-Wl,-T" in f for f in env.get("LINKFLAGS", [])):
    print("Warning! '-Wl,-T' option for specifying linker scripts is deprecated. "
          "Please use 'board_build.ldscript' option in your 'platformio.ini' file.")

#
# Default targets
#

Default([target_buildprog, target_size])


import os
import yaml
import csv
import json
import shlex
from shutil import copyfile

try:
    import xml.etree.cElementTree as ET
except ImportError:
    import xml.etree.ElementTree as ET

from util.pathlib import PathMgr
from util.pathfilter import FilterPathUtil
from task.codelintmodel import CodeLintModel
from util.subprocc import SubProcController
from util.logutil import LogPrinter

class Detekt(CodeLintModel):
    def __init__(self, params):
        CodeLintModel.__init__(self, params)
        self.sensitive_word_maps = {"detekt": "Tool", "Detekt": "Tool"}

    def analyze(self, params):
        """
        分析执行函数
        :param params:
        :return:
        """
        source_dir = params.source_dir
        work_dir = os.getcwd()
        enabled_rules = params.rules
        filter_mgr = FilterPathUtil(params)
        error_output = os.path.join(work_dir, "detekt-checkstyle.xml")
        envs = os.environ
        detekt_home = envs.get("DETEKT_171_HOME")
        pos = len(source_dir) + 1

        self.scan(
            params, detekt_home, report_type="xml", report_path=error_output, config=self.config(params, detekt_home)
        )

        raw_warning = ET.ElementTree(file=error_output)
        issues = []
        for f in raw_warning.iter(tag="file"):
            path = f.attrib.get("name")[pos:]
            if filter_mgr.should_filter_path(path):
                continue
            for error in f:
                line = int(error.attrib.get("line"))
                column = int(error.attrib.get("column"))
                tmp = error.attrib.get("source").split(".")
                rule = tmp[1]
                # 项目使用自己的规则配置文件，这里当项目指定了配置文件时候不做规则过滤
                if "DETEKT_PLUGINS_CONFIG" in envs and rule not in enabled_rules:
                    continue
                msg = error.attrib.get("message")
                issues.append({"path": path, "rule": rule, "msg": msg, "line": line, "column": column})
        LogPrinter.debug(issues)
        return issues

    def config(self, params, detekt_home):
        """

        :param params:
        :param detekt_home:
        :return:
        """
        source_dir = params.source_dir
        work_dir = params.work_dir
        enabled_rules = params.rules
        envs = os.environ

        # enable rules
        config_file = os.path.join(work_dir, "detekt-config.yml")
        if os.path.exists(config_file):
            os.remove(config_file)
        if enabled_rules:
            copyfile(os.path.join(detekt_home, "detekt-config-example.yml"), config_file)
            config = open(config_file)
            content = yaml.load(config, Loader=yaml.FullLoader)
            for item in content.values():
                if not isinstance(item, dict):
                    continue
                for rule in enabled_rules:
                    if rule in item:
                        item.get(rule)["active"] = True
            config = open(config_file, "w")
            yaml.dump(content, config, default_flow_style=False)
            config.close()
        else:
            copyfile(os.path.join(detekt_home, "puppy-default-detekt-config.yml"), config_file)

        # 设置第三方规则集jar配置
        user_plugins_config = envs.get("DETEKT_PLUGINS_CONFIG", None)
        if user_plugins_config:
            user_plugins_config_path = os.path.join(source_dir, user_plugins_config)
            # 若是配置文件，则提取出来
            # 修改为直接调用该config文件
            if os.path.isfile(user_plugins_config_path):
                config_file = user_plugins_config_path
            else:
                # 若是配置字符串
                user_config = json.loads(user_plugins_config)
                content = yaml.load(open(config_file))
                content.update(user_config)
                config = open(config_file, "w")
                # content中可能部分为unicode编码，dump时候可能会报错，这里是
                yaml.safe_dump(content, config, default_flow_style=False)
                config.close()

        return config_file

    def scan(self, params, detekt_home, report_type, report_path, source_dir=None, config=None, rule=None):
        """

        :param params:
        :param detekt_home:
        :param report_type:
        :param report_path:
        :param source_dir:
        :param config:
        :param rule:
        :return:
        """
        source_dir = source_dir if source_dir else params.source_dir
        path_include = params.path_filters.get("wildcard_inclusion", [])
        path_exclude = params.path_filters.get("wildcard_exclusion", [])
        envs = os.environ
        path_mgr = PathMgr()

        scan_cmd = [
            "java",
            "-jar",
            os.path.join(detekt_home, "detekt.jar"),
            "-i",
            source_dir,
            "--report",
            "%s:%s" % (report_type, report_path),
            "--debug",
            "--parallel",
        ]

        # 查看工具libs下有没有plugins第三方规则集jar包
        # 相关的第三方规则集配置写在detekt-config-example.yml中
        jars_list = path_mgr.get_dir_files(os.path.join(detekt_home, "libs"), ".jar")
        # 若客户指定代码库中的第三方规则集jar包, 支持输入目录或者文件，使用;隔开
        user_plugins = envs.get("DETEKT_PLUGINS", None)
        if user_plugins:
            user_plugins_list = [os.path.join(source_dir, path) for path in user_plugins.split(";")]
            for path in user_plugins_list:
                if os.path.isdir(path):
                    jars_list.extend(path_mgr.get_dir_files(path, ".jar"))
                elif os.path.isfile(path) and path.endswith(".jar"):
                    jars_list.append(path)
        if jars_list:
            scan_cmd.append("--plugins")
            scan_cmd.append(";".join(jars_list))

        # 设置规则配置
        if config:
            scan_cmd.extend(["--config", config])
        if rule:
            scan_cmd.extend(["--run-rule", rule])

        # 设置额外的detekt参数
        # 比如--jvm-target --classpath
        detekt_option_params = envs.get("DETEKT_OPTION_PARAMS", None)
        if detekt_option_params:
            scan_cmd.extend(shlex.split(detekt_option_params))

        # 设置路径过滤
        if path_exclude:
            scan_cmd.extend(
                ["--excludes", '"%s"' % (",".join(["**" + path.replace("*", "**") for path in path_exclude]))]
            )
        if path_include:
            scan_cmd.extend(
                ["--includes", '"%s"' % (",".join(["**" + path.replace("*", "**") for path in path_include]))]
            )

        self.print_log("scan cmd: %s" % " ".join(scan_cmd))
        subProC = SubProcController(scan_cmd, cwd=source_dir)
        subProC.wait()
        rt = subProC.returncode
        if rt != 0:
            LogPrinter.error("工具分析异常，返回码为%d" % rt)

tool = Detekt

if __name__ == "__main__":
    pass


import logging
import os
import pickle
import torch
from torch.utils.data import TensorDataset
from data_loader import InputFeatures
from utils import load_tokenizer, Tasks
# from xlm_ra import get_intent_labels

logger = logging.getLogger(__name__)

# noinspection PyCallingNonCallable
def generate_alignment_pairs(args):

    # Experiments could be faster by caching the alignment data
    # cached_features_file = os.path.join(args.model_dir, 'alignment_{}_{}.bin'.format(mode, args.task))

    examples = []
    # intent_labels = get_intent_labels(args) if args.task != Tasks.PAWS_X.value else None
    for language in args.align_languages.split(","):
        with open(os.path.join(args.data_dir, args.task, language, "train.tsv"), "r", encoding="utf-8") as tar_f, \
             open(os.path.join(args.data_dir, args.task, "en", "train.tsv"), "r", encoding="utf-8") as eng_f:
            data = pickle.load(open(os.path.join(args.data_dir, args.task, "en", "train", "data.pkl"), "rb"))
            for target_line, english_line, label in zip(tar_f, eng_f, data['intent_labels']):
                examples.append(((target_line.strip(), english_line.strip()), label.strip()))
        # ------------------- This is the experimental XLM-RA from the Discussion section ------------------------
        # with open(os.path.join(args.data_dir, args.task, language, "dev.tsv"), "r", encoding="utf-8") as tar_f, \
        #      open(os.path.join(args.data_dir, args.task, "en", "dev.tsv"), "r", encoding="utf-8") as eng_f:
        #     for target_line, english_line in zip(tar_f, eng_f):
        #         examples.append((target_line.strip(), english_line.strip()))
        # with open(os.path.join(args.data_dir, args.task, language, "test.tsv"), "r", encoding="utf-8") as tar_f, \
        #      open(os.path.join(args.data_dir, args.task, "en", "test.tsv"), "r", encoding="utf-8") as eng_f:
        #     for target_line, english_line in zip(tar_f, eng_f):
        #         examples.append((target_line.strip(), english_line.strip()))
        # --------------------------------------------------------------------------------------------------------
        logger.info("Read %d lines...." % len(examples))

    examples = examples[1:] if args.task == Tasks.PAWS_X.value else examples
    tokenizer = load_tokenizer(args.model_name_or_path)
    pad_token_id = tokenizer.pad_token_id

    feats = []
    for ex_id, example in enumerate(examples):
        if ex_id % 5000 == 0:
            logger.info("Processed %d examples..." % ex_id)
        input_ids = []

        if args.task == Tasks.PAWS_X.value:
            for utterance in example[0]:
                utterance = utterance.split("\t")[1:3]
                if len(utterance[0]) == 0 or len(utterance[1]) == 0:
                    continue
                tokens = [tokenizer.tokenize(u) for u in utterance]
                ids = tokenizer.build_inputs_with_special_tokens(tokenizer.convert_tokens_to_ids(tokens[0]),
                                                                 tokenizer.convert_tokens_to_ids(tokens[1]))
                assert tokens is not None and len(ids) <= args.max_seq_len
                ids = ids + ([pad_token_id] * (args.max_seq_len - len(ids)))
                input_ids.append(ids)
        elif args.task in [Tasks.MTOD.value, Tasks.MTOP.value, Tasks.M_ATIS.value]:
            for utterance in example[0]:
                tokens = tokenizer.tokenize(utterance)
                ids = tokenizer.build_inputs_with_special_tokens(tokenizer.convert_tokens_to_ids(tokens))
                assert tokens is not None and len(ids) <= args.max_seq_len
                ids = ids + ([pad_token_id] * (args.max_seq_len - len(ids)))
                input_ids.append(ids)
        else:
            raise Exception("The task '%s' is not recognised!" % args.task)

        if len(input_ids) == 2:
            # feats.append(InputFeatures(input_ids, intent_labels.index(example[1]), None, None))
            feats.append(InputFeatures(input_ids, None, None, None))

    # Convert to Tensors and build dataset
    target_input_ids = torch.tensor([f.input_ids[0] for f in feats], dtype=torch.long)
    english_input_ids = torch.tensor([f.input_ids[1] for f in feats], dtype=torch.long)
    # labels = torch.tensor([f.class_label for f in feats], dtype=torch.long)
    # train_dataset = TensorDataset(target_input_ids, english_input_ids, labels)
    # assert len(target_input_ids) == len(english_input_ids) == len(labels)
    train_dataset = TensorDataset(target_input_ids, english_input_ids)
    assert len(target_input_ids) == len(english_input_ids)

    logger.info("Created %d train/align instances." % len(train_dataset))
    return train_dataset


import tensorflow as tf
from tensorflow.python.ops import tensor_array_ops, control_flow_ops
import numpy as np

class GeneratorPoint(object):
    def __init__(self, num_emb, batch_size, emb_dim, hidden_dim,
                 dec_sequence_length, start_token,
                 enc_sequence_length,
                 rand_unif_init_mag, trunc_norm_init_std, rand_norm_init_std, adagrad_init_acc=0.1,
                 learning_rate=0.01,
                 grad_clip=5.0,
                 use_coverage=False
                 ):
        self.num_emb = num_emb
        self.batch_size = batch_size
        self.emb_dim = emb_dim
        self.hidden_dim = hidden_dim
        self.dec_sequence_length = dec_sequence_length
        self.enc_sequence_length = enc_sequence_length
        self.start_token = start_token
        self.learning_rate = tf.Variable(float(learning_rate), trainable=False)
        self.adagrad_init_acc = adagrad_init_acc
        self.grad_clip = grad_clip

        self.rand_unif_init = tf.random_uniform_initializer(minval=-rand_unif_init_mag, maxval=rand_unif_init_mag)
        self.trunc_norm_init = tf.truncated_normal_initializer(stddev=trunc_norm_init_std)

        self.rand_norm_init = tf.random_normal_initializer(stddev=rand_norm_init_std)

        self.use_coverage = use_coverage

    def add_placeholders(self):
        self.src_seq_batch = tf.placeholder(tf.int32, shape=[None, None], name='enc_batch') # sequence of tokens given to encoder
        self.enc_padding_mask = tf.placeholder(tf.float32, [None, None], name='enc_padding_mask')
        self.seq_len_batch = tf.placeholder(tf.int32, shape=[None], name='enc_lens')
        self.trg_seq_batch = tf.placeholder(tf.int32, shape=[None, self.dec_sequence_length], name='target_batch') # sequence of tokens generated by generator
        self.dec_seq_batch = tf.placeholder(tf.int32, shape=[None, self.dec_sequence_length], name='dec_batch') # sequence of tokens generated by generator
        self.dec_padding_mask = tf.placeholder(tf.float32, [None, self.dec_sequence_length], name='dec_padding_mask')
        self.start_dec = tf.placeholder(tf.int32, shape=[None], name='dec_start')
        self.g_smax = tf.placeholder(tf.float32, shape=[None, self.dec_sequence_length, self.num_emb], name='gsmax')
        self.enc_batch_extend_vocab = tf.placeholder(tf.int32, shape=[None, None], name='enc_batch_extend_vocab')
        self.max_art_oovs = tf.placeholder(tf.int32, [], name='max_art_oovs')

    def add_encoder(self):
        """Add a single-layer bidirectional LSTM encoder to the graph.

        Args:
          encoder_inputs: A tensor of shape [batch_size, <=max_enc_steps, emb_size].
          seq_len: Lengths of encoder_inputs (before padding). A tensor of shape [batch_size].

        Returns:
          encoder_outputs:
            A tensor of shape [batch_size, <=max_enc_steps, 2*hidden_dim]. It's 2*hidden_dim because it's the concatenation of the forwards and backwards states.
          fw_state, bw_state:
            Each are LSTMStateTuples of shape ([batch_size,hidden_dim],[batch_size,hidden_dim])
        """
        with tf.variable_scope('encoder') as ecode_scope:
            # Add embedding matrix
            with tf.variable_scope('embedding') as eemb_scope:
                embedding = tf.get_variable('embedding', [self.num_emb, self.emb_dim], dtype=tf.float32,
                                            initializer=self.trunc_norm_init)
                                            # initializer=self.rand_norm_init)

                emb_enc_inputs = tf.transpose(tf.nn.embedding_lookup(embedding, self.src_seq_batch), perm=[1, 0, 2])
            # Add the encoder.
            cell_fw = tf.contrib.rnn.LSTMCell(self.hidden_dim, state_is_tuple=True, initializer=self.rand_unif_init) #initializer=self.rand_unif_init)
            cell_bw = tf.contrib.rnn.LSTMCell(self.hidden_dim, state_is_tuple=True, initializer=self.rand_unif_init) #initializer=self.rand_unif_init))

            # cell_fw = tf.contrib.rnn.LSTMCell(self.hidden_dim, state_is_tuple=True, initializer=self.rand_unif_init)
            # cell_bw = tf.contrib.rnn.LSTMCell(self.hidden_dim, state_is_tuple=True, initializer=self.rand_unif_init)

            (encoder_outputs, (fw_st, bw_st)) = tf.nn.bidirectional_dynamic_rnn(cell_fw, cell_bw, emb_enc_inputs,
                                                                                dtype=tf.float32,
                                                                                # sequence_length=self.seq_len_batch,
                                                                                # swap_memory=True,)
                                                                                time_major=True)
            encoder_outputs = tf.transpose(tf.concat(axis=2, values=encoder_outputs), perm=[1, 0, 2]) # concatenate the forwards and backwards states
            self.encoder_states = encoder_outputs
            # Our encoder is bidirectional and our decoder is unidirectional so we need to reduce the final encoder hidden state to the right size to be the initial decoder hidden state
            self._dec_in_state = bw_st # self._reduce_states(fw_st, bw_st)

    def linear(self, args, output_size, bias, bias_start=0.0, scope=None):
        """Linear map: sum_i(args[i] * W[i]), where W[i] is a variable.

        Args:
          args: a 2D Tensor or a list of 2D, batch x n, Tensors.
          output_size: int, second dimension of W[i].
          bias: boolean, whether to add a bias term or not.
          bias_start: starting value to initialize the bias; 0 by default.
          scope: VariableScope for the created subgraph; defaults to "Linear".

        Returns:
          A 2D Tensor with shape [batch x output_size] equal to
          sum_i(args[i] * W[i]), where W[i]s are newly created matrices.

        Raises:
          ValueError: if some of the arguments has unspecified or wrong shape.
        """
        if args is None or (isinstance(args, (list, tuple)) and not args):
            raise ValueError("`args` must be specified")
        if not isinstance(args, (list, tuple)):
            args = [args]

        # Calculate the total size of arguments on dimension 1.
        total_arg_size = 0
        shapes = [a.get_shape().as_list() for a in args]
        for shape in shapes:
            if len(shape) != 2:
                raise ValueError("Linear is expecting 2D arguments: %s" % str(shapes))
            if not shape[1]:
                raise ValueError("Linear expects shape[1] of arguments: %s" % str(shapes))
            else:
                total_arg_size += shape[1]

        # Now the computation.
        with tf.variable_scope(scope or "Linear"):
            matrix = tf.get_variable("Matrix", [total_arg_size, output_size])
            if len(args) == 1:
                res = tf.matmul(args[0], matrix)
            else:
                res = tf.matmul(tf.concat(axis=1, values=args), matrix)
            if not bias:
                return res
            bias_term = tf.get_variable(
                "Bias", [output_size], initializer=tf.constant_initializer(bias_start))
        return res + bias_term

    def attention(self, decoder_state, inp):
        """Calculate the context vector and attention distribution from the decoder state.

        Args:
            decoder_state: state of the decoder
            coverage: Optional. Previous timestep's coverage vector, shape (batch_size, attn_len, 1, 1).

        Returns:
            context_vector: weighted sum of encoder_states
            attn_dist: attention distribution
            coverage: new coverage vector. shape (batch_size, attn_len, 1, 1)
        """
        with tf.variable_scope("attention_compute"):
            attn_size = 2*self.hidden_dim
            # Reshape encoder_states (need to insert a dim)
            encoder_states = tf.expand_dims(self.encoder_states, axis=2)  # now is shape (batch_size, attn_len, 1, attn_size)
            # To calculate attention, we calculate v^T tanh(W_h h_i + W_s s_t + b_attn)
            # where h_i is an encoder state, and s_t a decoder state.
            # attn_vec_size is the length of the vectors v, b_attn, (W_h h_i) and (W_s s_t).
            # We set it to be equal to the size of the encoder states.
            attention_vec_size = attn_size

            # Get the weight matrix W_h and apply it to each encoder state to get (W_h h_i), the encoder features
            W_h = tf.get_variable("W_h", [1, 1, attn_size, attention_vec_size])
            encoder_features = tf.nn.conv2d(encoder_states, W_h, [1, 1, 1, 1], "SAME")  # shape (batch_size,attn_length,1,attention_vec_size)

            # Pass the decoder state through a linear layer (this is W_s s_t + b_attn in the paper)
            decoder_features = self.linear(decoder_state, attention_vec_size, True) # shape (batch_size, attention_vec_size)
            decoder_features = tf.expand_dims(tf.expand_dims(decoder_features, 1), 1) # reshape to (batch_size, 1, 1, attention_vec_size)

            # Calculate v^T tanh(W_h h_i + W_s s_t + b_attn)
            e = tf.reduce_sum(tf.tanh(encoder_features + decoder_features), [2, 3]) # calculate e

            # Calculate attention distribution
            attn_dist = tf.nn.softmax(e*self.enc_padding_mask) # masked_attention(e)

            masked_sums = tf.reduce_sum(attn_dist, axis=1)  # shape (batch_size)
            attn_dist /= tf.reshape(masked_sums, [-1, 1])  # re-normalize

            # Calculate the context vector from attn_dist and encoder_states
            context_vector = tf.reduce_sum(tf.reshape(attn_dist, [tf.shape(self.src_seq_batch)[0], -1, 1, 1]) * encoder_states, [1, 2]) # shape (batch_size, attn_size).
            context_vector = tf.reshape(context_vector, [-1, attn_size])
        with tf.variable_scope('pgen_compute'):
            p_gen = self.linear([context_vector, decoder_state.c, decoder_state.h, inp], 1, True) # a scalar
            p_gen = tf.sigmoid(p_gen)
        return context_vector, attn_dist, p_gen

    def calc_final_dist(self, vocab_dist, attn_dist, p_gen):
        """Calculate the final distribution, for the pointer-generator model

        Args:
          vocab_dists: The vocabulary distributions. List length max_dec_steps of (batch_size, vsize) arrays. The words are in the order they appear in the vocabulary file.
          attn_dists: The attention distributions. List length max_dec_steps of (batch_size, attn_len) arrays

        Returns:
          final_dists: The final distributions. List length max_dec_steps of (batch_size, extended_vsize) arrays.
        """
        with tf.variable_scope('final_distribution'):
            # Multiply vocab dists by p_gen and attention dists by (1-p_gen)
            vocab_dist *= p_gen
            attn_dist *= (1 - p_gen)

            # Concatenate some zeros to each vocabulary dist, to hold the probabilities for in-article OOV words
            extended_vsize = self.num_emb + self.max_art_oovs  # the maximum (over the batch) size of the extended vocabulary
            extra_zeros = tf.zeros((tf.shape(self.src_seq_batch)[0], self.max_art_oovs))
            vocab_dists_extended = tf.concat(axis=1, values=[vocab_dist, extra_zeros]) # list length max_dec_steps of shape (batch_size, extended_vsize)

            # Project the values in the attention distributions onto the appropriate entries in the final distributions
            # This means that if a_i = 0.1 and the ith encoder word is w, and w has index 500 in the vocabulary, then we add 0.1 onto the 500th entry of the final distribution
            # This is done for each decoder timestep.
            # This is fiddly; we use tf.scatter_nd to do the projection
            batch_nums = tf.range(0, limit=tf.shape(self.src_seq_batch)[0])  # shape (batch_size)
            batch_nums = tf.expand_dims(batch_nums, 1)  # shape (batch_size, 1)
            attn_len = tf.shape(self.enc_batch_extend_vocab)[1]  # number of states we attend over
            batch_nums = tf.tile(batch_nums, [1, attn_len])  # shape (batch_size, attn_len)
            indices = tf.stack((batch_nums, self.enc_batch_extend_vocab), axis=2)  # shape (batch_size, enc_t, 2)
            shape = [tf.shape(self.src_seq_batch)[0], extended_vsize]
            attn_dist_projected = tf.scatter_nd(indices, attn_dist, shape) # list length max_dec_steps (batch_size, extended_vsize)

            # Add the vocab distributions and the copy distributions together to get the final distributions
            # final_dists is a list length max_dec_steps; each entry is a tensor shape (batch_size, extended_vsize) giving the final distribution for that decoder timestep
            # Note that for decoder timesteps and examples corresponding to a [PAD] token, this is junk - ignore.
            final_dist = vocab_dists_extended + attn_dist_projected
            return final_dist

    def decoder_unit(self, inp, prev_state):
        cell_gen = tf.contrib.rnn.LSTMCell(self.hidden_dim, state_is_tuple=True, initializer=self.rand_unif_init, reuse=tf.AUTO_REUSE)#initializer=self.rand_unif_init)
        # cell_gen = tf.contrib.rnn.LSTMCell(self.hidden_dim, state_is_tuple=True, initializer=self.rand_unif_init)
        return cell_gen(inp, prev_state)
        # return tf.nn.xw_plus_b(cell_output, dout_w, dout_b), state

    def add_generator(self):
        with tf.variable_scope('generator'):
            with tf.variable_scope('embedding') as demb_scope:
                d_embedding = tf.get_variable('embedding', [self.num_emb, self.emb_dim], dtype=tf.float32,
                                              initializer=self.trunc_norm_init)
                                              # initializer=self.rand_norm_init)
                emb_dec_inputs = tf.transpose(tf.nn.embedding_lookup(d_embedding, self.dec_seq_batch), perm=[1, 0, 2])

            with tf.variable_scope('decoding') as dcode_scope:
                #initializer=self.rand_unif_init)
                g_predictions = []
                final_dists = []
                g_attentions = []
                g_pointprobs = []
                g_logits = []
                trg_unstackd = tf.unstack(emb_dec_inputs, axis=0)
                state = self._dec_in_state
                # for attention
                # batch_size = self.encoder_states.get_shape()[0].value
                attn_size = self.encoder_states.get_shape()[2].value
                context_vector = tf.zeros([tf.shape(self.src_seq_batch)[0], attn_size])

                lst_context_v = []
                lst_cell_out = []
                for i, inp in enumerate(trg_unstackd):
                    if i > 0:
                        dcode_scope.reuse_variables()
                    inp = self.linear([inp] + [context_vector], self.emb_dim, True)
                    cell_output, state = self.decoder_unit(inp, state)
                    context_vector, attn_dist, p_gen = self.attention(state, inp)
                    # with tf.variable_scope("output_projection"):
                    #     cell_output = self.linear([cell_output, context_vector], self.hidden_dim, True)
                    #     logits = tf.nn.xw_plus_b(cell_output, dout_w, dout_b)
                    # g_logits.append(logits)
                    # g_predictions.append(tf.nn.softmax(logits))
                    g_attentions.append(attn_dist)
                    g_pointprobs.append(p_gen)
                    lst_context_v.append(context_vector)
                    lst_cell_out.append(cell_output)
                    # final_dist = self.calc_final_dist(tf.nn.softmax(logits), attn_dist, p_gen)
                    # final_dists.append(final_dist)

            with tf.variable_scope("output_projection") as out_project_scope:
                dout_w = tf.get_variable('w_out', [self.hidden_dim, self.num_emb],
                                         dtype=tf.float32, initializer=self.rand_unif_init)
                                         # dtype=tf.float32, initializer=self.rand_norm_init)
                dout_b = tf.get_variable('b_out', [self.num_emb],
                                         dtype=tf.float32, initializer=self.rand_unif_init)
                                         # dtype=tf.float32, initializer=self.rand_norm_init)
                for i, (context_vector, cell_output, attn_dist, p_gen) in enumerate(
                        zip(lst_context_v, lst_cell_out, g_attentions, g_pointprobs)):
                    if i > 0:
                        out_project_scope.reuse_variables()
                    cell_output = self.linear([cell_output, context_vector], self.hidden_dim, True)
                    logits = tf.nn.xw_plus_b(cell_output, dout_w, dout_b)
                    vocab_dist = tf.nn.softmax(logits)
                    g_predictions.append(vocab_dist)
                    final_dist = self.calc_final_dist(vocab_dist, attn_dist, p_gen)
                    final_dists.append(final_dist)
            self.final_dists = final_dists
            self.g_predictions = tf.transpose(tf.stack(g_predictions, axis=0), perm=[1, 0, 2])

            # Calculate the loss per step
            # This is fiddly; we use tf.gather_nd to pick out the probabilities of the gold target words
            loss_per_step = []  # will be list length max_dec_steps containing shape (batch_size)
            batch_nums = tf.range(0, limit=tf.shape(self.src_seq_batch)[0])  # shape (batch_size)
            for dec_step, (dist, targets) in enumerate(zip(self.final_dists, tf.unstack(tf.transpose(self.trg_seq_batch)))):
                indices = tf.stack((batch_nums, targets), axis=1)  # shape (batch_size, 2)
                gold_probs = tf.gather_nd(dist, indices)  # shape (batch_size). prob of correct words on this step
                losses = -tf.log(gold_probs)
                loss_per_step.append(losses)
            log_perps = tf.add_n(loss_per_step)

            # log_perp_list = []
            # for dist, target, weight in zip(g_logits, tf.unstack(tf.transpose(self.trg_seq_batch)),
            #                                  tf.unstack(tf.transpose(self.dec_padding_mask))):
            #     target = tf.reshape(target, [-1]) # The indices of the target words. shape (batch_size)
            #     crossent = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=target, logits=dist)
            #     log_perp_list.append(crossent * weight)
            # log_perps = tf.add_n(log_perp_list)
            # log_perps /= tf.reduce_sum(self.dec_padding_mask, axis=1)

            self.train_loss = tf.reduce_sum(log_perps)
            self.train_loss /= tf.cast(tf.shape(self.trg_seq_batch)[0], log_perps.dtype)

            # self.train_loss = tf.contrib.legacy_seq2seq.sequence_loss(
            #     g_logits, tf.unstack(tf.transpose(self.trg_seq_batch)), tf.unstack(tf.transpose(self.dec_padding_mask)))

            tf.summary.scalar('genr_loss', self.train_loss, collections=['genr_summaries'])

            gen_o = tensor_array_ops.TensorArray(dtype=tf.float32, size=self.dec_sequence_length,
                                                 dynamic_size=False, infer_shape=True)
            gen_x = tensor_array_ops.TensorArray(dtype=tf.int32, size=self.dec_sequence_length,
                                                 dynamic_size=False, infer_shape=True)

            def _g_recurrence(i, x_t, h_tm1, gen_x, c_vector):
                with tf.variable_scope('embedding', reuse=True) as demb_scope:
                    xt_emb = tf.nn.embedding_lookup(d_embedding, x_t)
                with tf.variable_scope('decoding', reuse=True) as dcode_scope:
                    inp = self.linear([xt_emb, c_vector], self.emb_dim, True)
                    cell_output, h_t = self.decoder_unit(inp, h_tm1)
                    c_vector, attn_dist, p_gen = self.attention(h_t, inp)
                with tf.variable_scope("output_projection", reuse=True) as out_project_scope:
                    cell_output = self.linear([cell_output, c_vector], self.hidden_dim, True)
                    logits = tf.nn.xw_plus_b(cell_output, dout_w, dout_b)
                    vocab_dist = tf.nn.softmax(logits)
                    # vocab_dist = self.calc_final_dist(vocab_dist, attn_dist, p_gen)
                log_prob = tf.log(vocab_dist)
                next_token = tf.cast(tf.reshape(tf.multinomial(log_prob, 1), [tf.shape(self.src_seq_batch)[0]]),
                # next_token = tf.cast(tf.reshape(tf.multinomial(vocab_dist, 1), [tf.shape(self.src_seq_batch)[0]]),
                                     tf.int32)
                # gen_o = gen_o.write(i, tf.reduce_sum(tf.multiply(tf.one_hot(next_token, self.num_emb, 1.0, 0.0), tf.nn.softmax(logits)), -1))
                # gen_o = gen_o.write(i, tf.nn.softmax(logits))
                gen_x = gen_x.write(i, next_token)  # indices, batch_size
                return i + 1, next_token, h_t, gen_x, c_vector

            _, _, _, self.gen_x, _ = control_flow_ops.while_loop(
                cond=lambda i, _1, _2, _3, _4: i < self.dec_sequence_length,
                body=_g_recurrence,
                loop_vars=(tf.constant(0, dtype=tf.int32),
                           tf.fill(tf.shape(self.seq_len_batch), self.start_token),
                           self._dec_in_state, gen_x,
                           tf.zeros([tf.shape(self.src_seq_batch)[0], attn_size])))

            self.gen_x = tf.transpose(self.gen_x.stack(), perm=[1, 0])  # batch_size x seq_length
            # self.gen_o = tf.transpose(self.gen_o.stack(), perm=[1, 0, 2])  # batch_size x seq_length

    def generate(self, sess, batch, _returns=None):
        feed = self._make_feed_dict(batch)
        feed.update({self.enc_batch_extend_vocab: batch.enc_batch_extend_vocab, self.max_art_oovs: batch.max_art_oovs})
        if _returns is not None:
            return sess.run(_returns, feed)
        return sess.run(self.gen_x, feed)

    def _make_feed_dict(self, batch):
        """Make a feed dictionary mapping parts of the batch to the appropriate placeholders.

        Args:
          batch: Batch object
          just_enc: Boolean. If True, only feed the parts needed for the encoder.
        """
        feed_dict = {}
        feed_dict[self.src_seq_batch] = batch.enc_batch
        feed_dict[self.enc_padding_mask] = batch.enc_padding_mask
        feed_dict[self.seq_len_batch] = batch.enc_lens
        # feed_dict[self.dec_padding_mask] = batch.dec_padding_mask
        feed_dict[self.dec_seq_batch] = batch.dec_batch
        feed_dict[self.trg_seq_batch] = batch.target_batch
        feed_dict[self.dec_padding_mask] = np.ones_like(batch.target_batch)
        return feed_dict

    def train_step(self, sess, batch, to_return, global_step=None):
        feed = self._make_feed_dict(batch)
        feed.update({self.enc_batch_extend_vocab: batch.enc_batch_extend_vocab, self.max_art_oovs:batch.max_art_oovs})
        outputs = sess.run([self.train_updates, self.train_loss, global_step]+to_return, feed)
        return outputs

    def add_train_opt(self, global_step=None):
        g_params = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, 'generator')
        e_params = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, 'encoder')
        self.train_opt = self.g_optimizer(self.learning_rate) #, self.adagrad_init_acc)
        # self.train_opt = self.g_optimizer(self.adagrad_init_acc)
        self.train_grad, _ = tf.clip_by_global_norm(tf.gradients(self.train_loss, g_params+e_params), self.grad_clip)
        self.train_updates = self.train_opt.apply_gradients(zip(self.train_grad, g_params+e_params), global_step=global_step)

    def g_optimizer(self, *args, **kwargs):
        return tf.train.AdamOptimizer(*args, **kwargs)
        # return tf.train.AdagradOptimizer(*args, **kwargs)

    def _reduce_states(self, fw_st, bw_st):
        """Add to the graph a linear layer to reduce the encoder's final FW and BW state into a single initial state for the decoder. This is needed because the encoder is bidirectional but the decoder is not.

        Args:
          fw_st: LSTMStateTuple with hidden_dim units.
          bw_st: LSTMStateTuple with hidden_dim units.

        Returns:
          state: LSTMStateTuple with hidden_dim units.
        """
        hidden_dim = self.hidden_dim
        with tf.variable_scope('reduce_final_st'):
            # Define weights and biases to reduce the cell and reduce the state
            w_reduce_c = tf.get_variable('w_reduce_c', [hidden_dim * 2, hidden_dim], dtype=tf.float32,
                                         initializer=self.rand_unif_init)
            w_reduce_h = tf.get_variable('w_reduce_h', [hidden_dim * 2, hidden_dim], dtype=tf.float32,
                                         initializer=self.rand_unif_init)
            bias_reduce_c = tf.get_variable('bias_reduce_c', [hidden_dim], dtype=tf.float32,
                                            initializer=self.rand_unif_init)
            bias_reduce_h = tf.get_variable('bias_reduce_h', [hidden_dim], dtype=tf.float32,
                                            initializer=self.rand_unif_init)

            # Apply linear layer
            old_c = tf.concat(axis=1, values=[fw_st.c, bw_st.c])  # Concatenation of fw and bw cell
            old_h = tf.concat(axis=1, values=[fw_st.h, bw_st.h])  # Concatenation of fw and bw state
            new_c = tf.nn.relu(tf.matmul(old_c, w_reduce_c) + bias_reduce_c)  # Get new cell from old cell
            new_h = tf.nn.relu(tf.matmul(old_h, w_reduce_h) + bias_reduce_h)  # Get new state from old state
            return tf.contrib.rnn.LSTMStateTuple(new_c, new_h)  # Return new cell and state


import time
import uuid

class DoctorService(object):

    def __init__(self):
        # {doctor_id: {doctor_id, first_name, last_name}}
        d_id1 = str(uuid.uuid4())
        d_id2 = str(uuid.uuid4())
        d_id3 = str(uuid.uuid4())
        self._DOCTORS = {
            d_id1: {'id': d_id1, 'first_name': 'Julius', 'last_name': 'Hibbert'},
            d_id2: {'id': d_id2, 'first_name': 'Algemop', 'last_name': 'Krieger'},
            d_id3: {'id': d_id3, 'first_name': 'Nick', 'last_name': 'Riviera'}
        }

        # {appointment_id: {appointment_id, patient_first_name, patient_last_name, date, time, kind}}
        self._APPOINTMENTS = {}

        # {doctor_id: [appointment_id]}
        self._DOCTOR_APPOINTMENTS = {}

    def _validate_appointment_time(self, appointment_time):
        try:
            time.strptime(appointment_time, '%H:%M')
        except ValueError:
            raise Exception('Invalid appointment time format given')

        hour = int(appointment_time.split(':')[0])
        if hour < 8 or hour > 16:
            raise Exception('Invalid time for appointment given. Appointment can only be scheduled between hours are between 8 AM and 5 PM only.')

        mins = int(appointment_time.split(':')[1])
        if mins % 15 != 0:
            raise Exception('Invalid start time for appointment given. Start time must be 0 mins, 15 mins, 30 mins, or 45 mins past the hour')

    def _validate_doctor(self, doctor_id):
        if doctor_id not in self._DOCTORS:
            raise Exception('Invalid doctor given')

    def get_all_doctors(self):
        return self._DOCTORS

    def get_doctor_appointments_for_date(self, doctor_id, appointment_date):
        self._validate_doctor(doctor_id)
        appointments = []
        return [self._APPOINTMENTS[a_id] for a_id in self._DOCTOR_APPOINTMENTS.get(doctor_id, []) if appointment_date == self._APPOINTMENTS[a_id]['date']]

    def delete_doctor_appointment(self, doctor_id, appointment_id):
        self._validate_doctor(doctor_id)
        if appointment_id not in self._APPOINTMENTS:
            raise Exception('Invalid appointment given')

        del _APPOINTMENTS[appointment_id]
        del _DOCTOR_APPOINTMENTS[doctor_id]

    def add_doctor_appointment(self, doctor_id, patient_first_name, patient_last_name, appointment_date, appointment_time, appointment_kind):
        self._validate_doctor(doctor_id)

        if appointment_kind not in ['New Patient', 'Follow-up']:
            raise Exeption('Invalid appointment kind given')

        if not patient_first_name or not patient_last_name:
            raise Exception('Invalid patient name')

        self._validate_appointment_time(appointment_time)

         # TODO: validate time --- a time check to see if time is within business hours

        # ensure that patient doesn't already have an appointment on the same day
        appointments = self.get_doctor_appointments_for_date(doctor_id, appointment_date)

        for appointment in appointments:
            if appointment['patient_first_name'] == patient_first_name and appointment['patient_last_name'] == patient_last_name:
                raise Exception('Patient already has appointment for the given date')
            if appointment['time'] == appointment_time:
                raise Exception('Doctor already has another appointment at the same time')

        # add the appointment
        appointment_id = str(uuid.uuid4())
        self._APPOINTMENTS[appointment_id] = {
            'id': appointment_id,
            'patient_first_name': patient_first_name,
            'patient_last_name': patient_last_name,
            'date': appointment_date,
            'time': appointment_time,
            'kind': appointment_kind
        }
        return appointment_id


#!/usr/bin/env python3
"""
tsmerge does two separate tasks: merging and gathering TimeSeries
files.

The need to merge arises when a PDE solution is interrupted and
resumed. This results in the creation of two or more TimeSeries
covering (typically) disjoint time intervals. The merge combines the
distinct TimeSeries into a single TimeSeries housed in a single file
that contains all the time points of the merged series.

The need to gather arises when a solution is run in multiple processes
in the MPi environment. When this is done with a build of HDF5 that
doesn't support parallel HDF5, it is necessary to save the results
from each process in distinct files. This form is invonvenient for
analysis, and also for resuming an interrupted process. The gather
combines the results of all the files representing a single TimeSeries
into a single file.

tsmerge takes two types of commandline arguments: the output file
prefix, specified with the -o or --outfile option, and a list of input
file prefixes. The output is stored in a file named <prefix>s1r0.h5,
where >prefix> is the prefix specified with the --outfile
option. Input prefixes are of two types. For a series stored in a
single file (that would typically be named either <inprefix>s1r0.h5 of
<inprefix>MPI.h5), the commandline argument is just the prefix. If,
however, one of the TimeSeries to be merged is stored in process files
that need to be gathered, there is a special syntax:
<inprefix>s<n>@. For instance, 'bases4@' would be used for a series
with basename base stored in four files bases4r0.h5, bases4r1.h5,
bases4r2.h5, and bases4r3.h5.
"""
import sys
from argparse import ArgumentParser
from KSFD import TimeSeries, Gatherer, KSFDException
import numpy as np
from mpi4py import MPI
import petsc4py

def main():
    comm = MPI.COMM_WORLD
    if comm.size > 1:
        raise KSFDException(
            'tsmerge must be run sequentially.'
        )
    petsc4py.init()
    parser = ArgumentParser(description='Merge time series',
                            allow_abbrev=True)
    parser.add_argument('-o', '--outfile',
                        help='merged file basename')
    parser.add_argument('-s', '--start', type=float, default=0.0,
                        help='start time')
    parser.add_argument('-e', '--end', type=float,
                        help='end time')
    parser.add_argument('infiles', nargs='+', help='files to merge')
    parser.add_argument('-v', '--verbose', action='count')
    clargs = parser.parse_args()
    times = np.empty((0), dtype=float)
    steps = np.empty((0), dtype=int)
    files = np.empty((0), dtype=int)
    grid = None
    for f,name in enumerate(clargs.infiles):
        if clargs.verbose > 0:
            print('collecting times from {name}'.format(name=name),
                  flush=True)
        g = Gatherer(name)
        if grid is None:
            grid = g.grid
        st = g.sorted_times()
        if clargs.verbose > 1:
            print('times:', st, flush=True)
        steps = np.append(steps, g.steps())
        times = np.append(times, st)
        files = np.append(files, np.full_like(st, f, dtype=int))
        g.close()
    out = TimeSeries(clargs.outfile, grid, comm=MPI.COMM_SELF, mode='w')
    # order = times.argsort()
    # files = files[order]
    # steps = steps[order]
    # times = times[order]
    k = 0
    del out.tsf['/info']
    for f,name in enumerate(clargs.infiles):
        if clargs.verbose > 0:
            print('collecting data from {name}'.format(name=name),
                  flush=True)
        fmatch = files == f
        # forder = order[fmatch]
        fsteps = steps[fmatch]
        ftimes = times[fmatch]
        g = Gatherer(name)
        if '/info' not in out.tsf:
            g.tsf.copy(
                source='/info',
                dest=out.tsf,
                name='/info',
                shallow=False
            )
        for s in g:
            if clargs.verbose > 0:
                print(str(s.tsf), flush=True)
            for k,t in zip(fsteps,ftimes):
                if t < clargs.start or (clargs.end and t > clargs.end):
                    continue
                vals = s.retrieve_by_number(k)
                if clargs.verbose > 1:
                    print('point {k}, time {t}'.format(k=k, t=t),
                          flush=True)
                out.store_slice(s.ranges, vals, t)
        g.close()
    out.close()

if __name__ == "__main__":
    # execute only if run as a script
    main()


import json
from datetime import datetime

# Setup variables for calculation.
# Each is dictionary, accessed by the keys gotten from Elasticsearch query. Refer to the query json.

# Weight of alarm severity.
WEIGHT = {
    'critical': 250,
    'high': 40,
    'medium': 1,
    'low': 0.1,
    'info': 0,
    'socialmedia': 15,
}

# Threshold of alarm severity.
# Define here above which state the severity category flags as 0.
# E.g. multiply num of vulnerabilities by their weight until THRESHOLD, then just set 0.
THRESHOLD = {
    'critical': 50,
    'high': 125,
    'medium': 10000,
    'low': 30000,
    'info': 1,
    'socialmedia': 9,
}

# List of rules
# Hold a dictionary which must have a `text` member for displaying measures to increase score.
RULES = {
    'critical': {'text': 'Mindestens ein Alarm von sehr hoher Schwere ist aufgetreten.'},
    'high': {'text': 'Mindestens ein Alarm von hoher Schwere ist aufgetreten.'},
    'medium': {'text': 'Mindestens ein Alarm von mittlere Schwere ist aufgetreten.'},
    'low': {'text': 'Mindestens ein Alarm von geringer Schwere ist aufgetreten.'},
    'info': {'text': 'Mindestens ein informativer Alarm ist aufgetreten.'},
    'socialmedia': {'text': 'Es wurde Social-Media-Aktivität erkannt.'}
}

# Calculate the total weight by summing up the dictionary.
totalWeight = sum(WEIGHT.values())
# Initialize score as 0
alertscore = 0.0
# List of offended rules from RULES.
offendingRules = []

# Read the contents of the result from Elasticsearch API query
file = open("/core4s/scripts/Automation/score_calculation/alerts_buckets.json", "r")
# Load content into a json datastore
datastore = json.load(file)

# the information is under this key. It is a list of buckets by severity
# 1, 2, 3, 4, 5
# as defined in the query json.
severitybuckets = datastore['aggregations']['severity']['buckets']

# for each severity bucket
for bucket in severitybuckets:
    severity = bucket['key']  # find out exactly which bucket we are looking at
    numAlerts = bucket['doc_count']
    if numAlerts:
        offendingRules.append(RULES[severity])  # RULE offended, add it.
    if numAlerts < THRESHOLD[severity]:  # Threshold not exceeded
        temp = numAlerts / THRESHOLD[severity]  # Calulate fraction of threshold that was exceeded.
    else:
        temp = 1  # Threshold exceeded => must take whole weight into account.
    temp *= WEIGHT[severity]  # multiply the fraction by weight e.g. weighted arithmetic mean, step 1
    alertscore += temp  # add the product to the alertscore e.g. weighted arithmetic mean, step 2

# social media rule
with open("/core4s/scripts/Automation/score_calculation/social_media_count.json", "r") as fsocial:
    dsSocial = json.load(fsocial)
    numSocial = dsSocial['count']
    if numSocial:
        offendingRules.append(RULES['socialmedia'])
    if numSocial < THRESHOLD['socialmedia']:
        temp = numSocial / THRESHOLD['socialmedia']
    else:
        temp = 1
    temp *= WEIGHT['socialmedia']
    alertscore += temp

alertscore = alertscore / totalWeight  # weighted arithmetic mean, step 3
alertscore = 1 - alertscore  # turn the percentage of not fulfilled into fulfilled 100%-X
alertscore = round(alertscore * 100, 2)  # 0.0047234234 => 0.0047 => 0.47%

result = {
    "score_type": "alert_score",
    "value": alertscore,
    "timestamp": int(datetime.utcnow().timestamp()) * 1000,
    "rules": offendingRules
}

with open('/tmp/alerts.scores.json', 'w') as tmpAlerts:
    json.dump(result, tmpAlerts)

print(result['value'])


from math import ceil

def calcular(valor,expoente,base):
    if expoente == 0:
        return valor
    if expoente == 1:
        return valor*base
    else:
        return calcular(valor*base*base,expoente-2,base)

def main():
    base,expoente=input().split()
    print(calcular(1,int(expoente),int(base)))
main()


from pathlib import Path

import cv2
import os
import yaml

from handwritten_char_extraction.char_extractor import CharExtractor

cfg_path = open('cfg.yml')
cfg = yaml.load(cfg_path, Loader=yaml.FullLoader)

ce = CharExtractor()

if __name__ == "__main__":
    cv2.namedWindow("Img", cv2.WINDOW_GUI_EXPANDED | cv2.WINDOW_FREERATIO)
    img_path = Path(cfg['test_img'])
    img_path = os.path.join(os.getcwd(), img_path)
    img = cv2.imread(img_path, cv2.IMREAD_GRAYSCALE)

    out = ce.process(img)

    cv2.imshow("Img", out)

    cv2.waitKey()
    cv2.destroyAllWindows()


import itertools
import MDAnalysis as mda
import MDAnalysis.analysis.base
from MDAnalysis.analysis import distances
import numpy as np

class DistanceAnalysis(MDAnalysis.analysis.base.AnalysisBase):
    def __init__(self, ag1, ag2, **kwargs):
        super(DistanceAnalysis, self).__init__(
            ag1.universe.trajectory, **kwargs)
        self._ag1 = ag1
        self._ag2 = ag2

    def _prepare(self):
        self.result = []

    def _single_frame(self):
        self.result.append(distances.dist(self._ag1, self._ag2)[-1])

    def _conclude(self):
        self.result = np.array(self.result)

class COM_analysis(MDAnalysis.analysis.base.AnalysisBase):
    def __init__(self, ag, **kwargs):
        super(COM_analysis, self).__init__(ag.universe.trajectory, **kwargs)
        self._ag = ag

    def _prepare(self):
        self.result = []

    def _single_frame(self):
        self.result.append(self._ag.center_of_mass())

    def _conclude(self):
        self.result = np.array(self.result)

def dis(vector):
    dis = []
    for i, vec in enumerate(vector):
        dis.append(np.linalg.norm(vec))
    dis = np.array(dis)
    return dis

def chunked_iterable(iterable, size):
    it = iter(iterable)
    while True:
        chunk = tuple(itertools.islice(it, size))
        if not chunk:
            break
        yield chunk

def distance_Phe_His(pdb, traj):  # info = [pdb, traj, ]
    sim = mda.Universe(pdb, traj)

    # Select atoms
    His_ring = sim.select_atoms(
        'resid 163 and not backbone and not name CB and not name H*')
    Phe_ring = sim.select_atoms(
        'resid 140 and not backbone and not name CB and name C*')
    # Calculation
    COM_Phe = []
    for n in range(2):
        com2 = COM_analysis(Phe_ring[int(n*6):int(n*6+6)])
        com2.run()
        COM_Phe.append(com2.result[:])
    COM_Phe = np.array(COM_Phe)
    COM_His = []
    for n in range(2):
        com3 = COM_analysis(His_ring[int(n*5):int(n*5+5)])
        com3.run()
        COM_His.append(com3.result[:])
    COM_His = np.array(COM_His)
    vec_Phe_His_A = COM_Phe[0] - COM_His[0]
    vec_Phe_His_B = COM_Phe[1] - COM_His[1]
    dist = np.array([dis(vec_Phe_His_A)]+[dis(vec_Phe_His_B)])
    dist = dist.T
    return dist

def distance_Arg_Glu(pdb, traj):
    sim = mda.Universe(pdb, traj)
    # Select atoms
    N_Arg = sim.select_atoms('resid 4 and name NH*')
    Glu = sim.select_atoms('resid 290 and name OE*')
    # Calculation
    N_Arg = N_Arg[3] + N_Arg[2] + N_Arg[3] + N_Arg[2] + \
        N_Arg[1] + N_Arg[0] + N_Arg[1] + N_Arg[0]
    Glu = Glu[0] + Glu[0] + Glu[1] + Glu[1] + Glu[2] + Glu[2] + Glu[3] + Glu[3]
    dis = DistanceAnalysis(Glu, N_Arg)
    dis.run()
    dis = np.array([np.min(dis.result[:, 0:4], 1)] +
                   [np.min(dis.result[:, 4:8], 1)])
    dis = dis.T
    return dis

def distance_Gln_Arg(pdb, traj):
    sim = mda.Universe(pdb, traj)
    Gln = sim.select_atoms('resid 299 and (name OE1 or name HE22)')
    Arg = sim.select_atoms('resid 4 and (name H or name O)')
    dis = DistanceAnalysis(Gln, Arg)
    dis.run()
    dis = np.array([np.min(dis.result[:, 0:2], 1)] +
                   [np.min(dis.result[:, 2:4], 1)])
    dis = dis.T
    return dis

def distance_Leu_Phe(pdb, traj):
    sim = mda.Universe(pdb, traj)
    Leu141 = sim.select_atoms(
        'resid 141 and (name HD12 or name HD21 or name HB2)')
    Phe3 = sim.select_atoms('resid 3 and (name CD1 or name CZ or name CD2)')
    Leu = Leu141[1] + Leu141[2] + Leu141[0] + Leu141[4] + Leu141[5] + Leu141[3]
    Phe = Phe3[3] + Phe3[4] + Phe3[5] + Phe3[0] + Phe3[1] + Phe3[2]
    dis = DistanceAnalysis(Leu, Phe)
    dis.run()
    dis = np.array([np.min(dis.result[:, 0:3], 1)] +
                   [np.min(dis.result[:, 3:7], 1)])
    dis = dis.T
    return dis

def oxyanion_dis(pdb, traj, res1, res2):
    sim = mda.Universe(pdb, traj)
    # Select atoms
    NH1 = sim.select_atoms(f'resid {res1} and name H')
    NH2 = sim.select_atoms(f'resid {res2} and name H')

    # Calculation
    dis1 = DistanceAnalysis(NH1, NH2)
    dis1.run()
    dis = dis1.result[:]
    return dis

def Tyr_Xaa_His(pdb, traj):
    sim = mda.Universe(pdb, traj)
    Tyr_OH = sim.select_atoms('resid 161 and name HH')
    His163_ND1 = sim.select_atoms('resid 163 and name ND1')
    dis1 = DistanceAnalysis(Tyr_OH, His163_ND1)
    dis1.run()
    dis = dis1.result[:]
    return dis

def Glu166_His172(pdb, traj):
    sim = mda.Universe(pdb, traj)
    Glu166 = sim.select_atoms('resid 166 and (name OE1 or name OE2)')
    His172 = sim.select_atoms('resid 172 and name NE2')
    His = His172[0] + His172[0] + His172[1] + His172[1]
    dis1 = DistanceAnalysis(Glu166, His)
    dis1.run()
    dis = np.array([np.min(dis1.result[:, 0:2], 1)] +
                   [np.min(dis1.result[:, 2:4], 1)])
    dis = dis.T
    return dis


####################迭代器###########################
class Fibs:
    def __init__(self, n = 25):
        self.a = 0;
        self.b = 1;
        self.n = n;
    def __iter__(self):     #重写__iter__和__next__即重新实现for循环
        return  self;
    def __next__(self):
        if self.b > self.n:
            raise  StopIteration("结束");
        a = self.a;
        b = self.b;
        self.a = b;
        self.b = a + b;
        return  self.a;

####################测试实例#########################
string = "hello";
it = iter(string);
while True:
    try:
        item = next(it);
    except StopIteration:
        break;
    print(item);
print("");

fibs = Fibs();
for fib in fibs:
    print(fib);
print("");

fibs = Fibs(10);
for fib in fibs:
    print(fib);


from config.conf import Suite

__author__ = "luxu"
from lib.funcslib import Ioput
Ioput.input("file",  r"CDdatil")
from lib.testsam import *
from flow.biportal.cdbase import CountData结算明细
file=Ioput.output("file")

class CaseSample首页(FunctionSample):

    for f in Get.out(file,  "Detail"):
        if Suite.table_data:
            exec("""
def test_{1}__表格数据(self):
   Ioput.function_name(self.__class__.__name__)
   self.首页_table({0})
               """.format(f, f.get("buttontext"))
             )
        if Suite.xlsx_data:
            exec("""
def test_{1}_导出数据(self):
   Ioput.function_name(self.__class__.__name__)
   self.首页_table({0})
               """.format(f, f.get("buttontext"))
         )

# class CaseSample详情(FunctionSample):
#     for f in Get.out(Ioput.output("file"), "Detail"):
#         exec("""
# def test_{0}(self):
#     Ioput.function_name(self.__class__.__name__)
#     self.详情_table({1})
#             """.format(f.get("casename"), f)
#              )

Test1=type("Test结算明细_首页", (CaseSample首页,), {"obj":CountData结算明细})

# for c in Get.out(file, ("tabletype", "Usertypedata")):
#     exec("""
# class Test用户统计详情_{0}_{2}(CaseSample详情):
#
#         obj=User用户统计详情
#         timetype="{1}"
#         Usertype="{2}"
#
#     """.format(c.get("casename"), c.get("tt"), c.get("buttontext"))
#          )

if __name__ == "__main__":
    unittest.main()


import copy

import numpy as np

from room import Room

def load_input(input_path, part2=False):
    char_list = []
    length = None
    with open(input_path) as file_obj:
        for line in file_obj:
            chars = list(line[:-1])
            char_list.append(chars)
    if part2:
        extra_lines = [
            list('  #D#C#B#A#'),
            list('  #D#B#A#C#'),
        ]
        char_list = char_list[:-2] + extra_lines + char_list[-2:]
    length = len(char_list[0])
    for chars in char_list:
        if len(chars) < length:
            chars += ['#'] * (length - len(chars))
    char_array = np.array(char_list)
    char_array[2:, :2] = '#'
    return char_array

def day23a(input_path, use_solution=False):
    """Return the minimum energy needed to get amphipods to the end state."""
    array = load_input(input_path)
    room = Room(array)
    # for testing purposes, optionally run the solution provided for the test
    # case instead of searching the solution space
    if use_solution:
        run_test_case_solution(room)
        return room.energy_used
    else:
        return run_room(room, energy_map=dict())

def run_room(room, energy_map):
    """Recursively move amphipods as allowable and fill out the energy map.

    The energy map has room states as keys and minimum energy from that state
    to the ending state as values.
    """
    if room.complete:
        return room.energy_used
    if room.to_tuple() in energy_map:
        return energy_map[room.to_tuple()] + room.energy_used
    min_energy = np.inf
    possible_moves = get_possible_moves_to_hallway(room)
    for move in possible_moves:
        room_copy = copy.deepcopy(room)
        room_copy.move(*move)
        run_hallway_movers(room_copy)
        energy = run_room(room_copy, energy_map)
        if energy:
            min_energy = min(min_energy, energy)
    energy_map[room.to_tuple()] = min_energy - room.energy_used
    return min_energy

def run_hallway_movers(room):
    """Iteratively try to move any amphipods from the hallway to their column.

    Continue until no amphipods are able to move from the hallway.
    """
    any_success = True
    while any_success:
        any_success = False
        for amph in room.hallway_movers:
            # try moving the the innermost position available
            for row in range(room.array.shape[0] - 2, 0, -1):
                dest = (row, amph.home_col)
                if room.array[dest] == '.' and room.check_move(amph, dest):
                    any_success = True
                    room.move(amph, dest)
                    break
    return any_success

def get_possible_moves_to_hallway(room):
    """Find all possible moves that move an amphipod from a room to the hallway.

    :return: tuple (amphipod coords, destination coords, number of steps)
    """
    possible_moves = []
    for amph in room.room_movers:
        for dest in room.open_coords:
            # ignore non-hallway destinations
            if dest[0] != 1:
                continue
            if room.check_move(amph, dest):
                possible_moves.append((amph.coords, dest))
    return possible_moves

def run_test_case_solution(room, max_moves=None):
    """Run the example solution given in the problem.

    This set of moves gives the optimal energy for part 1.
    If this test doesn't pass, then something is very wrong.
    """
    moves = [
        [(2, 7), (1, 4)],
        [(2, 5), (1, 6)],
        [(1, 6), (2, 7)],
        [(3, 5), (1, 6)],
        [(1, 4), (3, 5)],
        [(2, 3), (1, 4)],
        [(1, 4), (2, 5)],
        [(2, 9), (1, 8)],
        [(3, 9), (1, 10)],
        [(1, 8), (3, 9)],
        [(1, 6), (2, 9)],
        [(1, 10), (2, 3)],
    ]
    if max_moves is None:
        max_moves = len(moves)
    num_moves = 0
    for origin, dest in moves:
        amph = room.get_amph(origin)
        room.move(amph, dest)
        num_moves += 1
        if num_moves >= max_moves:
            return

def test23a():
    assert 12521 == day23a('test_input.txt', use_solution=True)
    assert 12521 == day23a('test_input.txt', use_solution=False)

def day23b(input_path):
    """Return the minimum energy needed to get amphipods to the end state.

    Larger state map for part 2.
    """
    array = load_input(input_path, part2=True)
    room = Room(array)
    min_energy = run_room(room, energy_map=dict())
    return min_energy

def test23b():
    assert 44169 == day23b('test_input.txt')

if __name__ == '__main__':
    test23a()
    print('Day 23a:', day23a('day23_input.txt'))
    test23b()
    print('Day 23b:', day23b('day23_input.txt'))


import docopt
from sklearn.neighbors import KNeighborsClassifier
import numpy as np
import pandas as pd
import matplotlib.pylab as plt
import itertools
import os #filesystem reads
from helpers import DataAnalyzer

#Crossvalidation:
from sklearn.model_selection import train_test_split
#multiprocessing
import multiprocessing

from functools import partial

#--------------------------------------------------------

DATA_FOLDER = "../data/trainsequences/"
CLASSIFY_FOLDER = "C:/Users/Ruben/Dropbox/Coding/GIT/Thesis/trainer/data/CLASSIFY"

ITERATIONS = 400
TEST_SIZE_PERCENT = 0.3

#--------------------------------------------------------

# Helper functions:

#extra arguments for the multiprocessing:
class WithExtraArgs(object):
  def __init__(self, func, *args):
    self.func = func
    self.args = args
  def __call__(self, idx):
    self.func(idx, *self.args)

# get all the data files from the directory
def getDataFileNames(dataType, movement = "", dataFolder = DATA_FOLDER):
  files = os.listdir(dataFolder);
  output = []
  for file in files:
    if dataType in file and movement in file:
      output.append(file)
  return output

#------ algorithm ----------
analyzer = DataAnalyzer.DataAnalyzer()

def alg(idx, data_data, data_labels):

  training_data, test_data, training_labels, test_labels = train_test_split(data_data, data_labels, test_size=TEST_SIZE_PERCENT)

  #------------ TRAINING -----------------

  #print("train label size:", len(training_labels))
  #print("train data size:", len(training_data))

  training_matrix = np.zeros((len(training_data),len(training_data)))

  #Enhanced method for creating the  DTW Similarity matrix. Now only computes the first half of the matrix
  for n in itertools.combinations(np.arange(len(training_data)), 2):
    calculation = analyzer.DTWSimilarity(training_data[n[0]], training_data[n[1]])
    training_matrix[n[0], n[1]] = calculation
    training_matrix[n[1], n[0]] = calculation

  model = KNeighborsClassifier()

  model.fit(training_matrix, training_labels)

  #----------- TESTING -------------------

  #test_data = []
  #test_labels = []
  #test_data_length = []

  #print("test data size:", len(test_data))
  #print("test label size:", len(test_labels))

  test_error = []
  for index, data in enumerate(test_data):
      row = []
      for secondData in training_data:
        row.append(analyzer.DTWSimilarity(data, secondData))

      #print("AffinityProp prediction for " + str(test_labels[index]) + " = " + str(model.predict([row])))
      if test_labels[index] != model.predict([row]):
        test_error.append(test_labels[index])
        #print("wrong prediction for " + str(test_labels[index]) + " = " + str(model.predict([row])))

  print((len(test_data)-len(test_error))/len(test_data))
  #glob_correct.append((len(test_data)-len(test_error))/len(test_data))
#CONTINUE WORKING HERE @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ GLOB_CORRECT STILL UNDEFINED

  return (len(test_data)-len(test_error))/len(test_data)

  '''

  for index, t in enumerate(test_data):

    test_matrix = []

    for data in t:
      row = []
      for secondData in t:
        row.append(analyzer.DTWSimilarity(data, secondData))
      test_matrix.append(row)

    print("AffinityProp prediction for " + str(test_labels[index]) + " = " + str(model.predict(test_matrix)))

  '''

#start the loop:

# ------------------- MAIN ------------------------------------

if __name__ ==  '__main__':

  print("Doing " + str(ITERATIONS) + " iterations with a " + str(TEST_SIZE_PERCENT) + " test ratio")

  # -- training --

  data_data = []
  data_labels = []
  data_data_length = []

  files = getDataFileNames("") #<-- Filter for ex: 'training' for only training data
  for trainingFile in files:
    dataFile = pd.read_csv(DATA_FOLDER + trainingFile, header = 0)
    #data = [dataFile['alpha'], dataFile['beta'], dataFile['gamma'], dataFile['accX'], dataFile['accY'], dataFile['accZ']]

    #dataset already normalized, so not necessary anymore
    #dataFile = analyzer.normalize(dataFile)
    #dataFile = analyzer.autoCorrelate(dataFile)

    data_data.append(dataFile)
    if "updown" in trainingFile:
      data_labels.append("updown")
    elif "leftright" in trainingFile:
      data_labels.append("leftright")
    elif "rotateclock" in trainingFile:
      data_labels.append("rotateclockwise")
    elif "square" in trainingFile:
      data_labels.append("square")
    elif "rest" in trainingFile:
      data_labels.append("rest")

  print("label size:", len(data_data))
  print("data size:", len(data_labels))
  print("---------")

  manager = multiprocessing.Manager()
  glob_correct = manager.list([])

  pool = multiprocessing.Pool()
  correct = pool.map(WithExtraArgs(alg, data_data, data_labels), range(ITERATIONS))

  print(correct)
  print("Mean: " + str(np.mean(glob_correct)))

#all data files loaded, now separate as training & test:


# List: ordered, mutable, allows duplicates
# Can contain different data types

import sys
from copy import deepcopy
import itertools

def empty():
    e = []
    assert len(e) == 0
    e = list() # alt syntax
    assert len(e) == 0

def creation():
    a = ['a', 2, 'c', 'c']
    assert a[0] == 'a'

def preallocate():
    a = [False] * 5
    assert a == [False, False, False, False, False]

def mutate():
    # single element
    a = [1, 2, 3, 4]
    a[1] = 2.2
    assert a[1] == 2.2

    # multiple elements
    a[1:2] = ['one', 'two']
    assert a[1] == 'one'
    assert a[2] == 'two'

    # truncate to [1, 0]
    a[1:] = [0]
    assert len(a) == 2

def find():
    a = [1, 2, 3, 4, 5]
    assert 1 in a
    assert 0 not in a

def append():
    # +=
    a = [1, 2]
    a += [3]
    assert a == [1, 2, 3]

    # append method
    a.append(4)
    assert a == [1, 2, 3, 4]

def insert():
    a = ['a', 'b', 'd']
    a.insert(2, 'c')
    assert a == ['a', 'b', 'c', 'd']

def concatenate():
    a = [1, 2]
    b = [3]
    c = a + b
    assert c == [1, 2, 3]

    a.extend(b)
    assert a == [1, 2, 3]

def unpack():
    a = [1, 2, 3]
    x, y, z = a
    assert x == 1
    assert y == 2
    assert z == 3

def unpackRemainder():
    a = [1, 2, 3]
    x, *y = a
    assert x == 1
    assert y == [2, 3]

def funcArg(a, b, c):
    assert a == 1
    assert b == 2
    assert c == 3

def unpackFuncArg():
    a = [1, 2, 3]
    funcArg(*a)

def remove():
    a = ['a', 'b', 'c']
    a.remove('b')
    assert a == ['a', 'c']

def pop():
    # remove index
    a = [1, 2, 3, 4]
    a.pop(2)
    assert a == [1, 2, 4]

    # remove end
    a.pop()
    assert a == [1, 2]

def clear():
    a = [1, 2]
    a.clear()
    assert len(a) == 0

def comprehension():
    # list = [expr for item in <iter> if [condition]]

    # square numbers in range
    y = [x ** 2 for x in range(9)]
    assert y == [0, 1, 4, 9, 16, 25, 36, 49, 64]

    # get all strings containing 'a':
    strs = ['ab', 'cd', 'ea', 'zz']
    y = [x for x in strs if 'a' in x]
    assert y == ['ab', 'ea']

    # get odd numbers
    n = list(range(11))
    y = [x for x in n if x % 2]
    assert y == [1, 3, 5, 7, 9]

    # convert strings to uppercase
    s = ['ab', 'cD', 'EF']
    u = [x.upper() for x in s]
    assert u == ['AB', 'CD', 'EF']

    # get public methods for list
    [x for x in dir([]) if '_' not in x]

    # (alt) get public methods for list
    [x for x in dir([]) if not x.startswith('_')]

def shallowCopy():
    a = [1, 2, 3]
    b = a.copy()
    c = a[:]
    assert a is not b
    assert a is not c

    # note: list1 = list2 creates a reference
    d = a
    assert a is d

def deepCopy():
    a = [1, 2, 3]
    b = deepcopy(a)
    assert a == b

def replicate():
    a = [1, 'b']
    a *= 3
    assert a == [1, 'b', 1, 'b', 1, 'b']

def bytes():
    a = [1, 2, 3, 4, 5]
    bytes = sys.getsizeof(a)
    assert bytes == 96

def index():
    # Find first element and return its index
    a = [4, 3, 2, 3]
    idx = a.index(3)
    assert idx == 1

    # Throws ValueError if item is not in container
    # a.index('fake_index')

def sort():
    a = [4, 2, 1, 3]
    a.sort()
    assert a == [1, 2, 3, 4]

def unique():
    # get unique elements in a list using set() constructor
    a = [2, 1, 2, 4, 4]
    unique = list(set(a))
    assert unique == [1, 2, 4]

def flatten():
    # flatten 2D list (matrix) using chain()
    matrix = [ [2, 3], [4, 5], [1, 2] ]

    result = list(itertools.chain(*matrix))
    assert result == [2, 3, 4, 5, 1, 2]

def main():
    empty()
    creation()
    preallocate()
    mutate()
    find()
    append()
    insert()
    concatenate()
    unpack()
    unpackRemainder()
    unpackFuncArg()
    remove()
    pop()
    clear()
    comprehension()
    shallowCopy()
    deepCopy()
    replicate()
    bytes()
    index()
    sort()
    unique()
    flatten()

if __name__ == '__main__':
    main()
    print('Tests passed!')


# -*- coding: utf-8 -*-

from os import listdir
from os.path import exists, join, splitext
from re import compile, search, sub
from shutil import copy, move

exts = [".c", ".h", ".y", ".def"]
#exts = [".py"]

def get_indent(line):
    s = []
    for c in line:
        if c != " ":
            return "".join(s)
        s.append(c)

def parse_args(line):
    args = []

    level = 0
    s = []
    for c in line:
        if c == "(":
            if 0 < level:
                s.append(c)
            level += 1
        elif level == 1:
            if c == ")":
                args.append("".join(s))
                return args
            elif c == ",":
                args.append("".join(s))
                s = []
            else:
                s.append(c)
        elif 0 < level:
            if c == ")":
                level -= 1
            s.append(c)

def replace_for_dir(dirname):
    for name in listdir(dirname):
        ext = splitext(name)[1]
        if ext in exts:
            path = join(dirname, name)
            orig = path + "~"
            if not exists(orig):
                copy(path, orig)

            with open(orig) as in_:
                tmp = path + ".tmp"
                with open(tmp, "w") as out:
                    for line in in_:
                        line = line.replace("YogMisc_check_encoding", "YogMisc_check_Encoding")
                        out.write(line)
            move(tmp, path)

def main():
    for dir in ["src", "src/gc", "include/yog", "tests", "tests/encoding"]:
        replace_for_dir(dir)

if __name__ == "__main__":
    main()

# vim: tabstop=4 shiftwidth=4 expandtab softtabstop=4