import os
import zipfile
from functools import partial
import nltk
import requests
import numpy as np
from ipywidgets import widgets
from collections import namedtuple
import logging
import tqdm
from typing import List, Optional, Dict, Tuple

def download_dataset() -> None:
    """
    This func downloads the dataset and unzip to a local dir.
    """
    folder_path = os.environ.get("IR1_DATA_PATH")
    if not folder_path:
        folder_path = "./datasets/"
    os.makedirs(folder_path)

    file_location = os.path.join(folder_path, "cacm.zip")

    if not os.path.exists(file_location):

        url = "https://surfdrive.surf.nl/files/index.php/s/M0FGJpX2p8wDwxR/download"

        with open(file_location, "wb") as handle:
            print(f"Downloading file from {url} to {file_location}")
            response = requests.get(url, stream=True)
            for data in response.iter_content():
                handle.write(data)
            print("Finished downloading file")

    if not os.path.exists(os.path.join(folder_path, "train.txt")):
        # unzip file
        with zipfile.ZipFile(file_location, "r") as zip_ref:
            zip_ref.extractall(folder_path)

def read_cacm_docs(root_folder="./datasets/"):
    """
    Reads in the CACM documents. The dataset is assumed to be in the folder "./datasets/" by default
    Returns: A list of 2-tuples: (doc_id, document), where 'document' is a single string created by
        appending the title and abstract (separated by a "\n").
        In case the record doesn't have an abstract, the document is composed only by the title
    """
    file_dir = root_folder + "cacm.all"
    keeping = False
    temp = ""
    doc_list = []

    with open(file_dir) as cacm_file:
        for line in cacm_file:
            line_begin = line[0:2]

            if line_begin == ".I":
                doc_index = line.split(" ")[1].replace("\n", "")
                temp = ""

            elif line_begin == ".T":
                keeping = True

            elif line_begin == ".W":
                keeping = True
                temp += "\n "

            elif line_begin in [".B", ".A", ".N", ".K"]:
                keeping = False

            elif line_begin not in [".I", ".T", ".W", ".B", ".A", ".N", ".X", ".K"]:
                if keeping:
                    temp += line
            else:
                doc_list.append((doc_index, temp))
                keeping = False

    return doc_list

def read_queries(root_folder="./datasets/"):
    """
    Reads in the CACM queries. The dataset is assumed to be in the folder "./datasets/" by default
    Returns: A list of 2-tuples: (query_id, query)
    """
    file_dir = root_folder + "query.text"
    keeping = False
    temp = ""
    query_list = []

    with open(file_dir, "r") as query_file:
        for line in query_file:
            line_begin = line[0:2]

            if line_begin == ".I":
                doc_index = line.split(" ")[1].replace("\n", "")
                temp = ""

            elif line_begin == ".W":
                keeping = True

            elif line_begin == ".A":
                keeping = False

            elif line_begin not in [".I", ".T", ".W", ".B", ".A", ".N", ".X", ".K"]:
                if keeping:
                    temp += line
            else:
                query_list.append((doc_index, temp))
                keeping = False

    return query_list

def load_stopwords(root_folder="./datasets/"):
    """
    Loads the stopwords. The dataset is assumed to be in the folder "./datasets/" by default

    :param root_folder:
    :return: A set of stopwords
    """
    file_dir = root_folder + "common_words"
    with open(file_dir, "r") as f:
        stopwords = [line.strip() for line in f]

    return set(stopwords)

def tokenize(text: str) -> List[str]:
    """
    Tokenizes the input text. Use the WordPunctTokenizer

    :param text: text - a string
    :return: a list of tokens
    """
    return nltk.WordPunctTokenizer().tokenize(text)

def stem_token(token):
    """
    Stems the given token using the PorterStemmer from the nltk library

    :param token: a single token
    :return: the stem of the token
    """
    ps = nltk.stem.PorterStemmer()
    return ps.stem(token)

def process_text(
    text: str,
    stem: bool = False,
    remove_stopwords: str = False,
    lowercase_text: str = False,
):
    """
    Given a string, this func tokenizes it and processes it according to the flags that you set.

    :param text: input text
    :param stem: to stem or not
    :param remove_stopwords: to remove stopwords or not
    :param lowercase_text: to lowercase the text or not
    :return:
    """
    tokens = []
    for token in tokenize(text):
        if remove_stopwords and token.lower() in stopwords:
            continue
        if stem:
            token = stem_token(token)
        if lowercase_text:
            token = token.lower()
        tokens.append(token)
    return tokens

def read_qrels(root_folder="./datasets/"):
    query_f = open(os.path.join(root_folder, "qrels.text"), "r")
    query_dic = {}

    for line in query_f:
        q_id, doc_id, _, _ = line.split()
        q_id = str(int(q_id))
        if q_id not in query_dic:
            query_dic[q_id] = []
        query_dic[q_id].append(doc_id)

    return query_dic


import os, shutil, time
import PySimpleGUI as sg

def automate(path=None):
    files = os.listdir(path)
    if files != []:
        for f in files:
            splitted_file_name = f.split('.')
            extention = splitted_file_name[-1]
            try:
                if not os.path.exists(f"{path}/{extention}"):
                    os.mkdir(f"{path}/{extention}")
                if extention in f:
                    src = f"{path}/{f}"
                    dst = f"{path}/{extention}"
                    shutil.move(src,dst)
            except FileExistsError or FileNotFoundError:
                print("Error")
    else:
        return "empty"

def get_files(path=None):
    files = os.listdir(path)
    if files != []:
        return files
    else:
        return "empty"

layout = [
    [sg.Text("Enter Directory"), sg.Input(key='path')],
    [sg.Button("Manage"), sg.Button("Show"), sg.FileBrowse("Browse")],
    [sg.Text("Press Manage to move",key='ptext'),sg.ProgressBar(100,size=(28,20),key='pbar',bar_color=('lightgreen','snow'))],
    [sg.Listbox([],size=(58,10),key='file-list',no_scrollbar=False),],
]

window = sg.Window("File Manager", layout, finalize=True,)

listbox = window['file-list']
progressbar = window['pbar']
progressbar.hide_row()
listbox.hide_row()

while True:
    event, values = window.read()

    if event in ("Exit", None):
        break

    input_path = values['path']
    if input_path is not None and input_path != " " and input_path != "":
        try:
            total_files = 100 if len(get_files(input_path)) < 100 else len(get_files(input_path))
        except:
            sg.popup_error("Enter a valid directory",title="Error")

        if event == "Show":
            if get_files(input_path) == "empty":
                sg.popup("Enter a valid directory.")
            else:
                listbox.unhide_row()
                listbox.update(get_files(input_path))

        if event == "Manage":
            if get_files(input_path) != "empty":
                progressbar.update(total_files)
                progressbar.unhide_row()
                automate(input_path)
                for i in range(total_files+1):
                    window['pbar'].UpdateBar(i)
                    window['ptext'].update(f"Automated {i}")
                    time.sleep(0.009)
                sg.popup("Files moved successfully.", title="Message")
            else:
                sg.popup_error("Enter a valid directory",title='Error')
    else:
        sg.popup_error("Enter a valid directory",title='Err')

window.close()


import datetime
import functools

from mypy_boto3_sns.client import SNSClient
from mypy_boto3_dynamodb.service_resource import Table
from threatexchange.signal_type.pdq import PdqSignal
from hmalib.common.models.pipeline import MatchRecord
import typing as t
import time

from threatexchange.signal_type.index import IndexMatch, SignalTypeIndex
from threatexchange.signal_type.signal_base import SignalType

from hmalib import metrics
from hmalib.common.logging import get_logger
from hmalib.common.mappings import INDEX_MAPPING
from hmalib.common.messages.match import BankedSignal, MatchMessage
from hmalib.common.configs.fetcher import (
    ThreatExchangeConfig,
    AdditionalMatchSettingsConfig,
)
from hmalib.common.models.bank import BankMember, BanksTable
from hmalib.common.models.signal import ThreatExchangeSignalMetadata
from hmalib.indexers.metadata import (
    BANKS_SOURCE_SHORT_CODE,
    THREAT_EXCHANGE_SOURCE_SHORT_CODE,
    BaseIndexMetadata,
    BankedSignalIndexMetadata,
)

logger = get_logger(__name__)

PG_CONFIG_CACHE_TIME_SECONDS = 300

@functools.lru_cache(maxsize=128)
def _get_privacy_group_matcher_active(privacy_group_id: str, cache_buster) -> bool:
    config = ThreatExchangeConfig.get(privacy_group_id)
    if not config:
        logger.warning("Privacy group %s is not found!", privacy_group_id)
        return False

    logger.debug("matcher_active for %s is %s", privacy_group_id, config.matcher_active)
    return config.matcher_active

def get_privacy_group_matcher_active(privacy_group_id: str) -> bool:
    """
    Is this privacy group's matcher_active flag true? Entries in the internal
    cache are cleared every PG_CONFIG_CACHE_TIME_SECONDS seconds.

    Impl: the // is python's integer division operator. Threw me off. :)
    """
    return _get_privacy_group_matcher_active(
        privacy_group_id, time.time() // PG_CONFIG_CACHE_TIME_SECONDS
    )

@functools.lru_cache(maxsize=None)
def _get_all_matcher_active_privacy_groups(cache_buster) -> t.List[str]:
    configs = ThreatExchangeConfig.get_all()
    return list(
        map(
            lambda c: c.name,
            filter(
                lambda c: c.matcher_active,
                configs,
            ),
        )
    )

@functools.lru_cache(maxsize=None)
def _get_max_pdq_threshold_for_active_matcher_privacy_groups(
    cache_buster,
) -> int:
    active_pg_names = _get_all_matcher_active_privacy_groups(cache_buster)
    if not active_pg_names:
        return 0
    active_pdq_thresholds = [
        config.pdq_match_threshold
        for config in AdditionalMatchSettingsConfig.get_all()
        if config.name in active_pg_names
    ]
    if active_pdq_thresholds:
        return max(active_pdq_thresholds)
    # no custom threshold set for active privacy_groups
    return 0

def get_max_threshold_of_active_privacy_groups_for_signal_type(
    signal_type: t.Type[SignalType],
) -> int:
    if signal_type == PdqSignal:
        return _get_max_pdq_threshold_for_active_matcher_privacy_groups(
            time.time() // PG_CONFIG_CACHE_TIME_SECONDS
        )
    else:
        return 0

@functools.lru_cache(maxsize=128)
def _get_privacy_group_matcher_pdq_threshold(
    privacy_group_id: str, cache_buster
) -> int:
    config = AdditionalMatchSettingsConfig.get(privacy_group_id)
    if not config:
        logger.debug(
            "Privacy group %s does not have custom pdq_match_threshold. Using default defined in PDQ_CONFIDENT_MATCH_THRESHOLD",
            privacy_group_id,
        )
        return PdqSignal.PDQ_CONFIDENT_MATCH_THRESHOLD

    logger.debug(
        "pdq_match_threshold for %s is %s", privacy_group_id, config.pdq_match_threshold
    )
    return config.pdq_match_threshold

def get_privacy_group_matcher_pdq_threshold(
    privacy_group_id: str,
) -> int:
    """
    Does this privacy group's have a custom pdq threshold; if so what is it?
    otherwise return the default PDQ_CONFIDENT_MATCH_THRESHOLD.

    Entries in the internal cache are cleared
    every PG_CONFIG_CACHE_TIME_SECONDS seconds.

    ToDo this should be refactored into a signal angostic interface eventaully
        especially before we have another similarity based signal type in HMA

    Impl: the // is python's integer division operator. Threw me off. :)
    """
    return _get_privacy_group_matcher_pdq_threshold(
        privacy_group_id, time.time() // PG_CONFIG_CACHE_TIME_SECONDS
    )

class Matcher:
    """
    Match against any signal type defined on threatexchange and stored in s3.

    Once created, indexes used by this are cached on the index. Do not create
    multiple Matcher instances in the same python runtime for the same
    signal_type. This would take up more RAM than necessary.

    Indexes are pulled from S3 on first call for a signal_type.
    """

    def __init__(
        self,
        index_bucket_name: str,
        supported_signal_types: t.List[t.Type[SignalType]],
    ):
        self.index_bucket_name = index_bucket_name
        self.supported_signal_types = supported_signal_types
        self._cached_indexes: t.Dict[t.Type[SignalType], SignalTypeIndex] = {}

    def match(
        self, signal_type: t.Type[SignalType], signal_value: str
    ) -> t.List[IndexMatch[BaseIndexMetadata]]:
        """
        Returns MatchMessage which can be directly published to a queue.

        Note, this also filters out matches that are from datasets that have
        been de-activated.
        """
        index = self.get_index(signal_type)

        with metrics.timer(metrics.names.indexer.search_index):
            match_results: t.List[IndexMatch] = index.query(signal_value)

        if not match_results:
            # No matches found in the index
            return []

        return self.filter_match_results(match_results, signal_type)

    def filter_match_results(
        self, results: t.List[IndexMatch], signal_type: t.Type[SignalType]
    ) -> t.List[IndexMatch]:
        """
        For ThreatExchange, use the privacy group's matcher_active flag to
        filter out match results that should not be returned.

        If implementing a matcher for something other than threat exchange,
        consider extending this class and implementing your own.
        """

        # results is a list of match object references that live in any index
        # this method should not edit those objects directly as they could effect
        # subsequent calls made while the index is still in memory
        matches = results.copy()

        filtered_results = []
        for match in matches:
            match.metadata = [
                metadata_obj
                for metadata_obj in match.metadata
                if metadata_obj.get_source() == BANKS_SOURCE_SHORT_CODE
                or (
                    # If metadata obj is from threatexchange (one privacy group
                    # per metadata obj), check that it is active AND
                    # if the signal_type is PdqSignal that its
                    # distance is lesser than the optional pdq_threshold set for
                    # that pg.
                    metadata_obj.get_source() == THREAT_EXCHANGE_SOURCE_SHORT_CODE
                    and get_privacy_group_matcher_active(metadata_obj.privacy_group)
                    and (
                        signal_type != PdqSignal
                        or match.distance
                        <= get_privacy_group_matcher_pdq_threshold(
                            str(metadata_obj.privacy_group)
                        )
                    )
                )
            ]

            if len(match.metadata) != 0:
                filtered_results.append(match)

        return filtered_results

    def write_match_record_for_result(
        self,
        table: Table,
        signal_type: t.Type[SignalType],
        content_hash: str,
        content_id: str,
        match: IndexMatch[BaseIndexMetadata],
    ):
        """
        Write a match record to dynamodb. The content_id is not important to the
        matcher. So, the calling lambda is expected to pass on the content_id
        for match record calls.
        """
        # Write one record for TE and one for banks.. I am sure the logic can be
        # simplified. We can do filters instead of iterating with flags. But
        # umm, we can fix that later?
        bank_record_written = False
        te_record_written = False

        for metadata_obj in match.metadata:
            match_record_attributes = {
                "content_id": content_id,
                "signal_type": signal_type,
                "content_hash": content_hash,
                "updated_at": datetime.datetime.now(),
                "signal_source": metadata_obj.get_source(),
                "match_distance": int(match.distance),
            }

            if (
                metadata_obj.get_source() == THREAT_EXCHANGE_SOURCE_SHORT_CODE
                and not te_record_written
            ):
                match_record_attributes.update(
                    signal_id=metadata_obj.indicator_id,
                    signal_hash=metadata_obj.signal_value,
                )
                te_record_written = True

            elif (
                metadata_obj.get_source() == BANKS_SOURCE_SHORT_CODE
                and not bank_record_written
            ):
                match_record_attributes.update(
                    signal_id=metadata_obj.signal_id,
                    signal_hash=metadata_obj.signal_value,
                )
                bank_record_written = True

            MatchRecord(**match_record_attributes).write_to_table(table)

    @classmethod
    def write_signal_if_not_found(
        cls,
        table: Table,
        signal_type: t.Type[SignalType],
        match: IndexMatch,
    ):
        """
        Write the signal to the datastore. Only signals that have matched are
        written to the DB. The fetcher takes care of updating the signal with
        opinions or updates from the source.

        TODO: Move this out of matchers.

        This is not matcher specific functionality. Signals could benefit from
        their own store. Perhaps the API could be useful when building local
        banks. Who knows! :)
        """
        for signal in cls.get_te_metadata_objects_from_match(signal_type, match):
            if hasattr(signal, "write_to_table_if_not_found"):
                # only ThreatExchangeSignalMetadata has this method.
                # mypy not smart enough to auto cast.
                signal.write_to_table_if_not_found(table)  # type: ignore

    @classmethod
    def get_te_metadata_objects_from_match(
        cls,
        signal_type: t.Type[SignalType],
        match: IndexMatch[BaseIndexMetadata],
    ) -> t.List[ThreatExchangeSignalMetadata]:
        """
        See docstring of `write_signal_if_not_found` we will likely want to move
        this outside of Matcher. However while the MD5 expansion is still on going
        better to have it all in once place.
        Note: changes made here will have an effect on api.matches.get_match_for_hash
        """
        return [
            ThreatExchangeSignalMetadata(
                signal_id=str(match_object.indicator_id),
                privacy_group_id=match_object.privacy_group,
                updated_at=datetime.datetime.now(),
                signal_type=signal_type,
                signal_hash=match_object.signal_value,
                tags=list(match_object.tags),
            )
            for match_object in match.metadata
            if match_object.get_source() == THREAT_EXCHANGE_SOURCE_SHORT_CODE
        ]

    def get_index(self, signal_type: t.Type[SignalType]) -> SignalTypeIndex:
        """
        If cached, return an index instance for the signal_type. If not, build
        one, cache and return.
        """

        max_custom_threshold = (
            get_max_threshold_of_active_privacy_groups_for_signal_type(signal_type)
        )
        index_cls = self._get_index_for_signal_type_matching(
            signal_type, max_custom_threshold
        )

        # Check for signal_type in cache AND confirm said index class type is
        # still correct for the given [optional] max_custom_threshold
        if not signal_type in self._cached_indexes or not isinstance(
            self._cached_indexes[signal_type], index_cls
        ):
            with metrics.timer(metrics.names.indexer.download_index):
                self._cached_indexes[signal_type] = index_cls.load(
                    bucket_name=self.index_bucket_name
                )

        return self._cached_indexes[signal_type]

    @classmethod
    def _get_index_for_signal_type_matching(
        cls, signal_type: t.Type[SignalType], max_custom_threshold: int
    ):
        indexes = INDEX_MAPPING[signal_type]
        # disallow empty list
        assert indexes
        if len(indexes) == 1:
            # if we only have one option just return
            return indexes[0]

        indexes.sort(key=lambda i: i.get_index_max_distance())

        for index_cls in indexes:
            if max_custom_threshold <= index_cls.get_index_max_distance():
                return index_cls

        # if we don't have an index that supports max threshold
        # just return the one if the highest possible max distance
        return indexes[-1]

    def publish_match_message(
        self,
        content_id: str,
        content_hash: str,
        matches: t.List[IndexMatch],
        sns_client: SNSClient,
        topic_arn: str,
    ):
        """
        Creates banked signal objects and publishes one message for a list of
        matches to SNS.
        """
        banked_signals = []

        for match in matches:
            for metadata_obj in match.metadata:
                if metadata_obj.get_source() == THREAT_EXCHANGE_SOURCE_SHORT_CODE:
                    banked_signal = BankedSignal(
                        str(metadata_obj.indicator_id),
                        str(metadata_obj.privacy_group),
                        str(metadata_obj.get_source()),
                    )
                    for tag in metadata_obj.tags:
                        banked_signal.add_classification(tag)

                    banked_signals.append(banked_signal)
                elif metadata_obj.get_source() == BANKS_SOURCE_SHORT_CODE:
                    banked_signal = BankedSignal(
                        metadata_obj.signal_id,
                        metadata_obj.bank_member_id,
                        metadata_obj.get_source(),
                    )

                    banked_signals.append(banked_signal)

        match_message = MatchMessage(
            content_key=content_id,
            content_hash=content_hash,
            matching_banked_signals=banked_signals,
        )

        sns_client.publish(TopicArn=topic_arn, Message=match_message.to_aws_json())


# 导入需要的库
from sklearn.preprocessing import LabelBinarizer
from sklearn.metrics import classification_report
from keras.models import load_model
from keras.datasets import cifar10
import numpy as np
import argparse
import glob
import os

# 构建命令行参数列表
ap = argparse.ArgumentParser()
ap.add_argument("-m", "--models", required=True, help="path to models directory")
args = vars(ap.parse_args())

# 加载数据集，然后缩放到0,1范围内
(testX, testY) = cifar10.load_data()[1]
testX = testX.astype("float") / 255.0
# 初始化数据集标签，并将Y标签从整数转为向量形式
labelNames = ["airplane", "automobile", "bird", "cat", "deer",
"dog", "frog", "horse", "ship", "truck"]
lb = LabelBinarizer()
testY = lb.fit_transform(testY)

# 收集模型路径，并初始化模型列表
modelPaths = os.path.sep.join([args["models"], "*.model"])
modelPaths = list(glob.glob(modelPaths))
models = []

# 从磁盘加载每一个模型,添加到model list列表中
for (i, modelPath) in enumerate(modelPaths):
    print("[INFO] loading model {}/{}".format(i + 1, len(modelPaths)))
    models.append(load_model(modelPath))

# 初始化预测列表
print("[INFO] evaluating ensemble...")
predictions = []

# 循环所有的模型
for model in models:
    # 使用列表中的每一个模型对testX数据集进行预测
    predictions.append(model.predict(testX, batch_size=64))

# 平均所有的模型预测结果，并做出来报告分析
predictions = np.average(predictions, axis=0)
print(classification_report(testY.argmax(axis=1),
predictions.argmax(axis=1), target_names=labelNames))


"""
Create tidy df from raw dfs of cell features, drug features, and drug responses.
Yitan provided the raw dfs and the data splits.
"""
from __future__ import print_function, division

import warnings
warnings.filterwarnings('ignore')

import os
import sys
import platform
from pathlib import Path
import psutil
import argparse
from datetime import datetime
from time import time
from pprint import pprint, pformat
from collections import OrderedDict
from glob import glob

import sklearn
import numpy as np
import pandas as pd

from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split

SEED = 42

# File path
file_path = Path(__file__).resolve().parent

# Utils
#utils_path = file_path / '../../utils'
#sys.path.append(str(utils_path))
#import utils
#from classlogger import Logger
import utils

utils_path = file_path / '../../utils'
sys.path.append(str(utils_path))
#import utils
from classlogger import Logger

# Path
OUTDIR = file_path/'../../data/yitan/Data/tidy'

def parse_args(args):
    parser = argparse.ArgumentParser(description="Create tidy df from raw (Yitan's) data.")
    parser.add_argument('--src', default='GDSC', type=str, help='Data source (default: GDSC).')
    # parser.add_argument('--fold', default=0, type=int, help='Fold (default: 0).')

    parser.add_argument('-cf', '--ccl_fea', nargs='+', default=['geneGE'], choices=['geneGE'],
                        help='Cell line features (default: `geneGE`).')
    parser.add_argument('-df', '--drg_fea', nargs='+', default=['DD'], choices=['DD'],
                        help='Drug features (default: `DD`).')

    parser.add_argument('-dk', '--drg_pca_k', default=None, type=int, help='Number of drug PCA components (default: None).')
    parser.add_argument('-ck', '--ccl_pca_k', default=None, type=int, help='Number of cell PCA components (default: None).')

    parser.add_argument('--n_jobs', default=4, type=int, help='Number of cpu workers (default: 4).')

    args = parser.parse_args(args)
    return args

def create_outdir(outdir, args):
    if any([True for i in [args['ccl_pca_k'], args['drg_pca_k']] if i is not None]):
        l = [args['src']] + args['ccl_fea'] + args['drg_fea'] + [f'drg_pca{drg_pca_k}'] + [f'ccl_pca{ccl_pca_k}']
    else:
        l = [args['src']] + args['ccl_fea'] + args['drg_fea']

    name_sffx = '.'.join( l )
    # outdir = Path(outdir) / (name_sffx + '_' + t)
    outdir = Path(outdir)/name_sffx
    os.makedirs(outdir)
    return outdir

def cnt_fea(df, fea_sep='_', verbose=True, logger=None):
    """ Count the number of features per feature type. """
    dct = {}
    unq_prfx = df.columns.map(lambda x: x.split(fea_sep)[0]).unique() # unique feature prefixes
    for prfx in unq_prfx:
        fea_type_cols = [c for c in df.columns if (c.split(fea_sep)[0]) in prfx] # all fea names of specific type
        dct[prfx] = len(fea_type_cols)

    if verbose and logger is not None:
        logger.info(pformat(dct))
    elif verbose:
        pprint(dct)

    return dct

def extract_subset_fea(df, fea_list, fea_sep='_'):
    """ Extract features based feature prefix name. """
    fea = [c for c in df.columns if (c.split(fea_sep)[0]) in fea_list]
    df = df[fea]
    return df

def run(args):
    src = args['src']
    ccl_fea_list = args['ccl_fea']
    drg_fea_list = args['drg_fea']
    n_jobs = args['n_jobs']
    drg_pca_k = args['drg_pca_k']
    ccl_pca_k = args['ccl_pca_k']
    fea_sep = '_'

    # =====================================================
    #       Logger
    # =====================================================
    run_outdir = create_outdir(OUTDIR, args)
    logfilename = run_outdir/'logfile.log'
    lg = Logger(logfilename)
    lg.logger.info(datetime.now())
    lg.logger.info(f'\nFile path: {file_path}')
    lg.logger.info(f'Machine: {platform.node()} ({platform.system()}, {psutil.cpu_count()} CPUs)')
    lg.logger.info(f'\n{pformat(args)}')

    # Dump args to file
    utils.dump_dict(args, run_outdir/'args.txt')

    # =====================================================
    #       Load data
    # =====================================================
    datadir = Path(file_path/'../../data/yitan/Data')
    ccl_folds_dir = Path(file_path/'../../data/yitan/CCL_10Fold_Partition')
    pdm_folds_dir = Path(file_path/'../../data/yitan/PDM_10Fold_Partition')
    fea_data_name = 'CCL_PDM_TransferLearningData_rmFactor_0.0_ddNorm_std.pkl'
    #fea_data_name = 'CCL_PDM_TransferLearningData_rmFactor_0.1_ddNorm_quantile.pkl'

    # Un-pickle files
    import _pickle as cp
    pkl_file = open(datadir/fea_data_name, 'rb')
    res = cp.load(pkl_file)
    ccl = cp.load(pkl_file)
    drg = cp.load(pkl_file)
    pkl_file.close()

    lg.logger.info('\n{}'.format('=' * 50))
    lg.logger.info('res: {}'.format(res.shape))
    lg.logger.info('ccl: {}'.format(ccl.shape))
    lg.logger.info('drg: {}'.format(drg.shape))

    # Update resp
    res = res.reset_index()
    res = res.rename(columns={'index': 'idx', 'ccl_name': 'cclname', 'SOURCE': 'src', 'area_under_curve': 'auc'})

    tmp = res.groupby('src').agg({'cclname': 'nunique', 'ctrpDrugID': 'nunique'}).reset_index()
    lg.logger.info(tmp)

    # =====================================================
    #       Prepare dfs for merging
    # =====================================================
    # Retain specific source
    lg.logger.info('\nFull dataset: {}'.format(res.shape))
    res = res[ res['src'].isin([src]) ]
    lg.logger.info('Only {}: {}'.format(src, res.shape))

    # Extract specific types of features
    lg.logger.info('\nExtract fea types ...')
    cnt_fea(ccl, logger=lg.logger);
    ccl = extract_subset_fea(df=ccl, fea_list=ccl_fea_list, fea_sep=fea_sep)
    cnt_fea(ccl, logger=lg.logger);

    cnt_fea(drg, logger=lg.logger);
    drg = extract_subset_fea(df=drg, fea_list=drg_fea_list, fea_sep=fea_sep)
    cnt_fea(drg, logger=lg.logger);

    # ---------------------------------------------
    #     PCA
    # ---------------------------------------------
    def pca_exp_var(pca, k):
        return np.sum(pca.explained_variance_ratio_[:k])

    def dump_pca_data(datadir, pca_x, k, fea_name):
        df = pd.DataFrame(pca_x.iloc[:, :k])
        df.to_csv(datadir/f'{fea_name}_pca{k}.csv', index=False)

    def pca_on_fea(df, fea_name, k, datadir):
        lg.logger.info(f'Calc PCA for {fea_name} features.')
        index = df.index
        n = 1400
        pca = PCA(n_components=n, random_state=SEED, svd_solver='auto')
        pca_x = pca.fit_transform(df)
        pca_x = StandardScaler().fit_transform(pca_x)
        pca_x = pd.DataFrame(pca_x, index=index,
                             columns=[f'{fea_name}_PC'+str(i+1) for i in range(n)])
        exp_var = pca_exp_var(pca, k)
        # dump_pca_data(datadir, pca_x=pca_x, k=k, fea_name=fea_name);
        lg.logger.info(f'{fea_name} PCA variance explained {exp_var:.4f} (k={k}).')
        return pca_x.iloc[:, :k]

    if drg_pca_k is not None:
        ccl = pca_on_fea(df=ccl, fea_name='ccl', k=ccl_pca_k, datadir=datadir)

    if ccl_pca_k is not None:
        drg = pca_on_fea(df=drg, fea_name='drg', k=drg_pca_k, datadir=datadir)

    # Bring the labels in, in order to merge on
    ccl = ccl.reset_index().rename(columns={'index': 'cclname'})
    drg = drg.reset_index().rename(columns={'index': 'ctrpDrugID'})

    # =====================================================
    #       Merge
    # =====================================================
    def merge_dfs(res_df, ccl_df, drg_df):
        """ Merge the following dfs: response, ccl fea, drug fea """
        mrg_df = pd.merge(res_df, ccl_df, on='cclname', how='inner')
        mrg_df = pd.merge(mrg_df, drg_df, on='ctrpDrugID', how='inner')
        return mrg_df

    lg.logger.info('\nMerge ...')
    mrg_df = merge_dfs(res, ccl, drg)
    lg.logger.info('mrg_df.shape: {}'.format(mrg_df.shape))

    # =====================================================
    #       Extract xdata, ydata, meta and  dump to file
    # =====================================================
    if any([True for i in [ccl_pca_k, drg_pca_k] if i is not None]):
        prfx = '.'.join( [src.lower()] + ccl_fea_list + drg_fea_list + [f'drg_pca{drg_pca_k}'] + [f'ccl_pca{ccl_pca_k}'] )
    else:
        prfx = '.'.join( [src.lower()] + ccl_fea_list + drg_fea_list )

    mrg_df.to_parquet(run_outdir/(prfx+'_data.parquet'), index=False)
    lg.logger.info('mrg_df usage: {:.2f} GB\n'.format(sys.getsizeof(mrg_df)/1e9))

    lg.kill_logger()

def main(args):
    args = parse_args(args)
    args = vars(args)
    ret = run(args)

if __name__ == '__main__':
    """ __name__ == '__main__' explained: www.youtube.com/watch?v=sugvnHA7ElY """
    main(sys.argv[1:])


from Transparency.Trainers.TrainerTransformer import go
from argparse import ArgumentParser
import os

def run_experiments(arg):
    result = go(arg)
    return result

#parser.add_argument("--csv_path",
#                    help = "Path to the csv containing the preprocessed information",
#                    type = str, default = )

if __name__ == "__main__":

    parser = ArgumentParser()

    parser.add_argument("--model_type",
                            choices= ["diversity_transformer", "vanilla_transformer"],
                            required=True, type=str)

    parser.add_argument("--dataset_name",
                        help = "Name of the dataset to train on",
                        required = True,type = str)
    parser.add_argument("--random-seed", help = "What random seed to use?",
                        default = 0, type = int)

    arg = parser.parse_args()
    run_experiments(arg)

"""
parser.add_argument("-e", "--num-epochs",
                dest="num_epochs",
                help="Number of epochs.",
                default=10, type=int)

parser.add_argument("-b", "--batch-size",
                dest="batch_size",
                help="The batch size.",
                default=4, type=int)

parser.add_argument("-l", "--learn-rate",
                dest="lr",
                help="Learning rate",
                default=0.0001, type=float)

#parser.add_argument("-T", "--tb_dir", dest="tb_dir",
#                    help="Tensorboard logging directory",
#                    default='./runs')

parser.add_argument("--max_pool", dest="max_pool",
                help="Use max pooling in the final classification layer.",
                action="store_true")

parser.add_argument("-E", "--embedding", dest="embedding_size",
                help="Size of the character embeddings.",
                default=300, type=int) # 128

parser.add_argument("-M", "--max", dest="max_length",
                help="Max sequence length. Longer sequences are clipped (-1 for no limit).",
                default=512, type=int)

parser.add_argument("-H", "--heads", dest="num_heads",
                help="Number of attention heads.",
                default=8, type=int)

parser.add_argument("-d", "--depth", dest="depth",
                help="Depth of the network (nr. of self-attention layers/ transformer blocks)",
                default=1, type=int) # 6

parser.add_argument("-r", "--random-seed",
                dest="seed",
                help="RNG seed. Negative for random",
                default=1, type=int)

parser.add_argument("--gradient_clipping",
                    dest="gradient_clipping",
                    help="Gradient clipping.",
                    default=1.0, type=float)

parser.add_argument("--diversity_transformer",
                    help="Add diveristy term to loss?",
                    default=False, type=bool)

parser.add_argument("--diversity_weight",
                    help="Diversity weight for the diveristy part of the loss",
                    default=0.5, type=float)

parser.add_argument("--delete_prop",
                    help="What proportion of the values with the heighest weight should be set to 0?",
                    default=0, type=float)

parser.add_argument("--dataset_name",
                help = "Name of the dataset to train on",
                required = True,type = str)

parser.add_argument("--lr-warmup",
            dest="lr_warmup",
            help="Learning rate warmup.",
            default=5_000, type=int)
"""


from datetime import datetime, date
from sqlite3.dbapi2 import Date

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions
from selenium.webdriver.support.wait import WebDriverWait

chrome_options = webdriver.ChromeOptions();
chrome_options.add_experimental_option("excludeSwitches",
['enable-automation']);

driver = webdriver.Chrome("C:\\Users\\kalmarevelin\\chromedriver.exe", options=chrome_options)
driver.get("http://www.learnwebservices.com/locations/?size=100")

def click_on_create_location():
    driver.find_element(By.ID, "create-location-link").click()

def wait_for_name_input():
    WebDriverWait(driver, 10).until(
        expected_conditions.visibility_of_element_located(
            (By.ID, "location-name")
        )
    )
    return driver.find_element(By.ID, "location-name")

def wait_for_coords_input():
    WebDriverWait(driver, 10).until(
        expected_conditions.visibility_of_element_located(
            (By.ID, "location-coords")
        )
    )
    return driver.find_element(By.ID, "location-coords")

def create_name():
    now = datetime.now()
    timestamp = datetime.timestamp(now)
    location_name = 'Ajax_' + str(timestamp)
    return location_name

def type_location_name():
    location_name = create_name()
    driver.find_element(By.ID, "location-name").send_keys(location_name)

def type_coords(coord1, coord2):
    driver.find_element(By.ID, "location-coords").send_keys(coord1, ",", coord2)

def create_new_location():
    driver.find_element(By.XPATH, "//*[@id='location-form']/input[1]").click()

def location_has_created():
    WebDriverWait(driver, 10).until(
        expected_conditions.text_to_be_present_in_element((By.ID, "message-div"),
                                                              "Location has created"))
    message = driver.find_element(By.ID, "message-div").text
    print(message)

def wait_for_location_in_table():
    location_name = create_name()
    WebDriverWait(driver, 10).until(
        expected_conditions.text_to_be_present_in_element((By.ID, "locations-table"),
                                                          "Ajax_"))

def assert_coords():
    coords = driver.find_element(By.XPATH, "//tr[td[contains(text(), 'Ajax_')]]/td[3]").text
    print(type(coords))
    #assert coords == 10.0, 10.0

def blank_name_message():
    WebDriverWait(driver, 10).until(
        expected_conditions.text_to_be_present_in_element((By.ID, "message-div"),
                                                              "must not be blank"))
    message = driver.find_element(By.ID, "message-div").text
    print(message)

def blank_coord_message():
    WebDriverWait(driver, 10).until(
        expected_conditions.text_to_be_present_in_element((By.ID, "message-div"),
                                                              "invalid coordinates"))
    message = driver.find_element(By.ID, "message-div").text
    print(message)

def invalid_coord_message():
    WebDriverWait(driver, 10).until(
        expected_conditions.text_to_be_present_in_element((By.ID, "message-div"),
                                                              "invalid coordinates"))
    message = driver.find_element(By.ID, "message-div").text
    print(message)

def click_on_cancel():
    driver.find_element(By.ID, "create-cancel-button").click()

click_on_create_location()
wait_for_name_input()
wait_for_coords_input()
create_name()
type_location_name()
type_coords(10, 10)
create_new_location()
location_has_created()
wait_for_location_in_table()
assert_coords()

#click_on_create_location()
#wait_for_name_input()
#wait_for_coords_input()
#type_coords(11, 11)
#create_new_location()
#blank_name_message()
#click_on_cancel()

click_on_create_location()
wait_for_name_input()
wait_for_coords_input()
create_name()
type_location_name()
create_new_location()
blank_coord_message()
click_on_cancel()

click_on_create_location()
wait_for_name_input()
wait_for_coords_input()
create_name()
type_location_name()
type_coords(10,  10)
invalid_coord_message()


import numpy as np
import cv2 as cv
from matplotlib import pyplot as plt
import calibration
from scipy import optimize

import scipy

imgL = cv.imread('imgL.jpeg')  # queryimage # left image
imgM = cv.imread('imgM.jpeg')  # trainimage # middle image
imgR = cv.imread('imgR.jpeg')  # trainimage # right image
imgMR = cv.imread('imgM.jpeg') # Copy of middle image to later reconstruct the object with using 3 images.
imgML = cv.imread('imgM.jpeg') # Copy of middle image to later reconstruct the object with using 3 images.

"""Explanation: L = Left, M = Middle, R = Right
    ML = Middle-Left, MR = Middle-Right"""

sift = cv.SIFT_create(nfeatures=0, nOctaveLayers=3, contrastThreshold=0.01, edgeThreshold=20, sigma=0.5)
# find the keypoints and descriptors with SIFT
[kpL, desL] = sift.detectAndCompute(imgL, None)
[kpM, desM] = sift.detectAndCompute(imgM, None)
[kpR, desR] = sift.detectAndCompute(imgR, None)
# Show keypoints:
img_kpM = imgM.copy()
img_kpL = imgL.copy()
img_kpR = imgR.copy()
img_kpM = cv.drawKeypoints(imgM, kpM,img_kpM) # Draw keypoints found by SIFT algorithm in the middle image.
img_kpL = cv.drawKeypoints(imgL,kpL, img_kpL) # Draw keypoints found by SIFT algorithm in the middle image.
img_kpR = cv.drawKeypoints(imgR, kpR,img_kpR) # Draw keypoints found by SIFT algorithm in the middle image.
plt.figure("Middle Image Keypoints")
plt.imshow(img_kpM)
plt.figure("Left Image Keypoints")
plt.imshow(img_kpL)
plt.figure("Right Image Keypoints")
plt.imshow(img_kpR)

# FLANN parameters
FLANN_INDEX_KDTREE = 1
index_params = dict(algorithm=FLANN_INDEX_KDTREE, trees=5) # was trees=5 # was trees 10
search_params = dict(checks=50)
flann = cv.FlannBasedMatcher(index_params, search_params)
matches_ML = flann.knnMatch(desL, desM, k=2)  # Find matches between images Left and Middle
matches_MR = flann.knnMatch(desM, desR, k=2)  # Find matches between images Middle and Right

ptsL = []
ptsML = []
ptsMR = []
ptsR = []
# ratio test as per Lowe's paper - for middle and left images
for i, (m, n) in enumerate(matches_ML):
    if m.distance < 0.8 * n.distance: # was 0.7 # was 0.99
      #if m.distance < 400:
        ptsML.append(kpM[m.trainIdx].pt)
        ptsL.append(kpL[m.queryIdx].pt)

# ratio test as per Lowe's paper - for middle and right images

for i, (m, n) in enumerate(matches_MR):
    if m.distance < 0.88 * n.distance: # was 0.7 # was 0.99
      #if m.distance < 400:
        ptsMR.append(kpM[m.queryIdx].pt)
        ptsR.append(kpR[m.trainIdx].pt)

ptsL = np.double(ptsL)
ptsML = np.double(ptsML)
ptsMR = np.double(ptsMR)
ptsR = np.double(ptsR)

"""Now we have the list of best matches from both the images. Let's find the Fundamental Matrix."""

F_ML, mask_ML = cv.findFundamentalMat(ptsML, ptsL, cv.FM_LMEDS,maxIters=50000,confidence = 0.99,ransacReprojThreshold = 3) # Find the Fundamental matrix that matches the middle and left images.
F_MR, mask_MR = cv.findFundamentalMat(ptsR, ptsMR, cv.FM_LMEDS,maxIters=50000,confidence = 0.99,ransacReprojThreshold = 3) # Find the Fundamental matrix that matches the right and middle images.

# We select only inlier points
ptsL = ptsL[mask_ML.ravel() == 1]
ptsML = ptsML[mask_ML.ravel() == 1]
ptsMR = ptsMR[mask_MR.ravel() == 1]
ptsR = ptsR[mask_MR.ravel() == 1]

"""Next we find the epilines. Epilines corresponding to the points in first image is drawn on second image.
So mentioning of correct images are important here.
We get an array of lines. So we define a new function to draw these lines on the images."""

def drawlines(img1, img2, lines, pts1, pts2):
    ''' img1 - image on which we draw the epilines for the points in img2
        lines - corresponding epilines '''
    pts1 = np.int32(pts1)
    pts2 = np.int32(pts2)
    obj = img1.shape
    r, c = obj[0], obj[1]
    # r,c = img1.shape
    # img1 = cv.cvtColor(img1,cv.COLOR_GRAY2BGR)
    # img2 = cv.cvtColor(img2,cv.COLOR_GRAY2BGR)
    for r, pt1, pt2 in zip(lines, pts1, pts2):
        color = tuple(np.random.randint(0, 255, 3).tolist())
        x0, y0 = map(int, [0, -r[2] / r[1]])
        x1, y1 = map(int, [c, -(r[2] + r[0] * c) / r[1]])
        img1 = cv.line(img1, (x0, y0), (x1, y1), color, 1)
        img1 = cv.circle(img1, tuple(pt1), 5, color, -1)
        img2 = cv.circle(img2, tuple(pt2), 5, color, -1)
    return img1, img2

# Find epilines corresponding to points in right image (second image) and
# drawing its lines on left image
lines1 = cv.computeCorrespondEpilines(ptsMR.reshape(-1, 1, 2), 2, F_MR)
lines1 = lines1.reshape(-1, 3)
img5, img6 = drawlines(imgR, imgMR, lines1, ptsR, ptsMR)
# Find epilines corresponding to points in left image (right image) and
# drawing its lines on right image
lines2 = cv.computeCorrespondEpilines(ptsR.reshape(-1, 1, 2), 1, F_MR)
lines2 = lines2.reshape(-1, 3)
img3, img4 = drawlines(imgMR, imgR, lines2, ptsMR, ptsR)
plt.figure("Epilines and matches between images")
plt.subplot(221), plt.imshow(img5)
plt.subplot(222), plt.imshow(img3)

lines3 = cv.computeCorrespondEpilines(ptsL.reshape(-1, 1, 2), 2, F_ML)
lines3 = lines3.reshape(-1, 3)
img7, img8 = drawlines(imgL, imgML, lines3, ptsL, ptsML)
# Find epilines corresponding to points in left image (first image) and
# drawing its lines on right image
lines4 = cv.computeCorrespondEpilines(ptsML.reshape(-1, 1, 2), 1, F_ML)
lines4 = lines4.reshape(-1, 3)
img9, img10 = drawlines(imgML, imgL, lines4, ptsML, ptsL)
plt.subplot(223), plt.imshow(img7)
plt.subplot(224), plt.imshow(img9)
#plt.show()

# Calibrating the camera:
C = calibration.calibrate()
ret,mtx,dist, rvecs,tvecs = C.get_calibration()
K=mtx

# Get P of left image:
E_ML = K.T@F_ML@K
R1_ML,R2_ML,t_ML= cv.decomposeEssentialMat(E_ML)
Rt_ML = np.column_stack((R1_ML,t_ML)) # R2 better 3D reproj
P_ML=np.matmul(K,Rt_ML)

# Get P of right image:
E_MR = K.T@F_ML@K
R1_MR,R2_MR,t_MR = cv.decomposeEssentialMat(E_MR)
Rt_MR = np.column_stack((R1_MR,t_MR))# R2 better 3D reproj
P_MR=np.matmul(K,Rt_MR)

# Get P of middle image:
Rt_M = np.array([[1,0,0,0],
              [0,1,0,0],
              [0,0,1,0]])
P_M = np.matmul(K,Rt_M)

#------------------------------------triangulation-------------------------------------------
#converting pts to floats
kpL_3D = np.ones((3, ptsL.shape[0]))
kpML_3D = np.ones((3, ptsML.shape[0]))
kpMR_3D = np.ones((3, ptsMR.shape[0]))
kpR_3D = np.ones((3, ptsR.shape[0]))
kpL_3D[0], kpL_3D[1] = ptsL[:, 0].copy(), ptsL[:, 1].copy()
kpML_3D[0], kpML_3D[1] = ptsML[:, 0].copy(), ptsML[:, 1].copy()
kpMR_3D[0], kpMR_3D[1] = ptsMR[:, 0].copy(), ptsMR[:, 1].copy()
kpR_3D[0], kpR_3D[1] = ptsR[:, 0].copy(), ptsR[:, 1].copy()

Points4D_ML = cv.triangulatePoints(P_M, P_ML, kpML_3D[:2], kpL_3D[:2])
Points4D_MR = cv.triangulatePoints(P_MR,P_M , kpMR_3D[:2], kpR_3D[:2])

# Normalize the points.
Points4D_ML /= Points4D_ML[3]
Points4D_MR /= Points4D_MR[3]

points_3d_ML = np.transpose(Points4D_ML[:3, :])
fig = plt.figure("3D Image Reconstruction Result")
ax = fig.add_subplot(1, 1, 1, projection='3d')
ax.scatter(points_3d_ML[:,0], points_3d_ML[:,1], points_3d_ML[:,2],color='blue')

points_3d_MR = np.transpose(Points4D_MR[:3, :])
scale_factor = 1
points_3d_MR *= scale_factor
ax.scatter(points_3d_MR[:,0], points_3d_MR[:,1], points_3d_MR[:,2],color='blue')
plt.show()

#---------------------------optimization-------------------------------------------

Points3D_ML_reproj = P_M[:3] @ Points4D_ML
Points3D_L_reproj = P_ML[:3] @ Points4D_ML

Points3D_MR_reproj = P_M[:3] @ Points4D_MR
Points3D_R_reproj = P_MR[:3] @ Points4D_MR

Points3D_ML_reproj /= Points3D_ML_reproj[2]
Points3D_L_reproj /= Points3D_L_reproj[2]

Points3D_MR_reproj /= Points3D_MR_reproj[2]
Points3D_R_reproj /= Points3D_R_reproj[2]

# Calculate reprojection error:
#reproj: [x-x_repoj,y-y_repoj] [pts1[0][0]-Points3D_L_back[0][0],pts1[0][1]-Points3D_L_back[0][1]]
# reproj_ML,J = cv.projectPoints(kpL_3D,R1_ML,t_ML,mtx,dist) #returns image points on image plane + jacobian
# #scipy minimize
#
# err_reproj_L=[]
# err_reproj_ML=[]
# err_reproj_MR=[]
# err_reproj_R=[]
#
# for i in range(kpL_3D.shape[1]):
#     err_reproj_L.append((abs(kpL_3D[0][i]-Points3D_L_reproj[0][i]))**2 + (abs(kpL_3D[1][i]-Points3D_L_reproj[1][i]))**2)
#     err_reproj_ML.append((abs(kpML_3D[0][i] - Points3D_ML_reproj[0][i])) ** 2 + (abs(kpML_3D[1][i] - Points3D_ML_reproj[1][i])) ** 2)
#     # kp[0] = x coordinate, kp[1] = y coordinate
#
#
# for i in range(kpR_3D.shape[1]):
#     err_reproj_R.append((abs(kpR_3D[0][i]-Points3D_R_reproj[0][i]))**2 + (abs(kpR_3D[1][i]-Points3D_R_reproj[1][i]))**2)
#     err_reproj_MR.append((abs(kpMR_3D[0][i] - Points3D_ML_reproj[0][i])) ** 2 + (abs(kpMR_3D[1][i] - Points3D_R_reproj[1][i])) ** 2)
#
# # Calculate Middle-Left loss function:
# err_L = sum(err_reproj_L)/err_reproj_L.__len__()
# err_ML = sum(err_reproj_ML)/err_reproj_ML.__len__()
# loss_ML = err_L + err_ML
#
# # Calculate Middle-Right loss function:
# err_R = sum(err_reproj_R)/err_reproj_R.__len__()
# err_MR = sum(err_reproj_MR)/err_reproj_MR.__len__()
# loss_MR = err_R + err_MR
#
#
# result= optimize.minimize(optimize.rosen,err_repoj_L,method='BFGS',tol=1e-5)
#print(result.x)

# Perform PCA analysis
    #mean = np.empty((0))
    #mean1, eigenvectors1, eigenvalues1 = cv.PCACompute2(data_pts1, mean1)
   # mean2, eigenvectors2, eigenvalues2 = cv.PCACompute2(data_pts2, mean2)


#!/usr/bin/env python3

import serial
import MySQLdb
import time
# import sys

# box_status="1:0:1:0:1:0:0:0:0:0"

# Open database connection
db = MySQLdb.connect("localhost","admin","vs12345","Vendigo_S1" )

# prepare a cursor object using cursor() method
cursor = db.cursor()
Bstatus = "SELECT BoxStatus FROM BoxStatus"

def process_db():
      try:
         # Execute the SQL command
         cursor.execute(Bstatus)
         global boxstatus
         boxstatus = ""
         #Fetching complete column from the table
         result = cursor.fetchall();
         result = list(result)
         index = 0
         status=['0','0','0','0','0','0','0','0','0','0']
         while index < 10:
            status[index] = list(result[index])
            index += 1
         for i in range (0,10):
            if status[i] == ['0']:
               # print("Box: " + str(i+1) + " Free")
               if i != 9:
                  boxstatus += "0:"
               else:
                  boxstatus += "0"
            else:
               # print("Box: " + str(i+1) + " Occupied")
               if i != 9:
                  boxstatus += "1:"
               else:
                  boxstatus += "1"
         boxstatus.strip()

         print("boxstatus_command = " + boxstatus)
         ser.write(bytes(boxstatus, 'utf-8'))
         # ser.write("1:1:1:0:0:0:1:1:1:1".encode('utf-8'))
         time.sleep(1)
         ser.flushOutput()
         # ser.write(b'1:1:1:0:0:0:1:1:1:1')
         # if(boxstatus == "0:0:0:0:0:0:0:0:0:0") ser.write(bytes(b'1111111111'))

      except Exception:
         exc_type, value, traceback = sys.exc_info()
         print("Failed with exception [%s]" % exc_type.__name__)
         # print("Issue with operation, Kindly Crosscheck Database script!!!!")

if __name__ == '__main__':
   while True:
    try:
      ser = serial.Serial('/dev/ttyUSB1', 115200, timeout=5)
      if ser.isOpen()==True:
         ser.reset_input_buffer()
         break
      ser = serial.Serial('/dev/ttyUSB0', 115200, timeout=5)
      if ser.isOpen()==True:
         ser.reset_input_buffer()
         break
    except serial.SerialException:
      print("Connecting with other port")

   while True:
        process_db()
        time.sleep(1)
        # if ser.in_waiting > 0:
        #     line = ser.readline().decode('utf-8').rstrip()
        #     print(line)
   # disconnect from server
   db.close()


"""
Main
"""
from flask import Flask
from flask import jsonify
from flask import request
from flask import render_template
from werkzeug.exceptions import NotFound

from views.products import products_app

app = Flask(__name__)
app.register_blueprint(products_app, url_prefix="/products")

def print_request():
    """
    print_request
    :return:
    """
    print("request:", request)
    print("headers", request.headers)

@app.get("/")
def get_root():
    """
    get root
    :return:
    """
    # print_request()
    print(request.args)
    # return {
    #     "request.args": request.args,
    #     "request.args['foo']": request.args["foo"],
    #     "request.args.get('foo')": request.args.get("foo"),
    #     "request.args.getlist('foo')": request.args.getlist("foo"),
    #     "request.args.getlist('spam')": request.args.getlist("spam"),
    #     "request.args.to_dict()": request.args.to_dict(),
    #     "request.args.to_dict(flat=False)": request.args.to_dict(flat=False),
    #     "message": "Hello!",
    # }
    return render_template("index.html")

@app.route("/hello/")
@app.route("/hello/<name>/")
def hello_world(name: str = None):
    """
    hello_world
    :param name:
    :return:
    """
    # print_request()
    if name is None:
        name = request.args.get("name", "world")
    return f"
Hello {name}!
"

@app.get("/items/")
def get_items():
    """
    get_items
    :return:
    """
    return jsonify(
        {
            "items": [
                {"id": 1},
                {"id": 2},
            ],
        },
        # spam="eggs",
    )

@app.get("/items/<int:item_id>/")
def get_item(item_id: int):
    """
    get_item
    :param item_id:
    :return:
    """
    return {"item": {"id": item_id}}

@app.get("/items/<item_id>/")
def get_item_as_string(item_id: str):
    """
    get_item_as_string
    :param item_id:
    :return:
    """
    return {"item_id": item_id.upper()}

@app.errorhandler(404)
def handle_404(error):
    """
    handle_404
    :param error:
    :return:
    """
    if isinstance(error, NotFound) and error.description != NotFound.description:
        return error
    return f"
eroror: {error}
"

if __name__ == "__main__":
    app.run(debug=True)


from django.urls import path, re_path
# from django.conf.urls import url
from .views import SampleTemplateView, MyCreateView, MyUpdateView, MyDeleteView
from .views import find
#from django.views.generic import TemplateView

urlpatterns = [
    path('sample/', SampleTemplateView.as_view(), name='index'),
    path('sample/create', MyCreateView.as_view(), name='create'),
    path('sample/find', find, name='find'),
    re_path('sample/edit/(?P<pk>\d+)$', MyUpdateView.as_view(), name='edit'),  #汎用viewでURLに値を渡すには、re_pathがいいかな？
    re_path('sample/delete/(?P<pk>\d+)$', MyDeleteView.as_view(), name='delete')
]


import pathlib

filePath = pathlib.Path("Days/day3/input.txt")

file = open(filePath, "r")
backpacks = list(map(lambda x: x.strip(), file.readlines()))

ascii_a = ord("a")
ascii_z = ord("z")
ascii_A = ord("A")
ascii_Z = ord("Z")

total = 0

for backpack in backpacks:
    middle = len(backpack) // 2
    container1 = backpack[0:middle]
    container2 = backpack[middle:len(backpack)]
    for item1 in container1:
        if (item1 not in container2):
            continue
        if (ascii_a <= ord(item1) <= ascii_z):
            total += ord(item1) - ascii_a + 1
            break
        else:
            total += ord(item1) - ascii_A + 27
            break

print(total)

total_2 = 0
for i in range(0, len(backpacks), 3):
   common = set(backpacks[i]).intersection(set(backpacks[i + 1])).intersection(set(backpacks[i + 2])).pop()

   if (ascii_a <= ord(common) <= ascii_z):
        total_2 += ord(common) - ascii_a + 1
   else:
        total_2 += ord(common) - ascii_A + 27

print(total_2)


from django.contrib.auth.models import Permission, User, Group
from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import SuspiciousOperation
from django.db import transaction
from django.db.models import Q, Count
from django.shortcuts import get_object_or_404
from django.utils.translation import ugettext_lazy as _
from django.views.decorators.cache import cache_page

from guardian.models import UserObjectPermission, GroupObjectPermission
from guardian.shortcuts import get_perms

from igdectk.rest.handler import *
from igdectk.rest.response import HttpResponseRest
from igdectk.module.manager import module_manager
from main.cursor import CursorQuery
from main.models import Entity, Profile

from .filters import GroupFilter, UserFilter
from .utils import get_permissions_for, prefetch_permissions_for

class RestPermission(RestHandler):
    regex = r'^permission/$'
    name = 'permission'

class RestPermissionType(RestPermission):
    regex = r'^type/$'
    suffix = 'type'

class RestPermissionUser(RestPermission):
    regex = r'^user/$'
    suffix = 'user'

class RestPermissionUserCount(RestPermissionUser):
    regex = r'^user/count/$'
    suffix = 'user/count/'

class RestPermissionUserSearch(RestPermissionUser):
    regex = r'^search/$'
    suffix = 'search'

class RestPermissionUserName(RestPermissionUser):
    regex = r'^username/(?P<username>[a-zA-Z0-9\.\-_]+)/$'
    suffix = 'username'

class RestPermissionUserNamePermission(RestPermissionUserName):
    regex = r'^permission/$'
    suffix = 'permission'

class RestPermissionGroup(RestPermission):
    regex = r'^group/$'
    suffix = 'group'

class RestPermissionGroupCount(RestPermission):
    regex = r'^count/$'
    suffix = 'count'

class RestPermissionGroupSearch(RestPermissionGroup):
    regex = r'^search/$'
    suffix = 'search'

class RestPermissionGroupId(RestPermissionGroup):
    regex = r'^(?P<grp_id>[0-9]+)/$'
    suffix = 'name'

class RestPermissionGroupIdUser(RestPermissionGroupId):
    regex = r'^user/$'
    suffix = 'user'

class RestPermissionGroupIdUserName(RestPermissionGroupIdUser):
    regex = r'^username/(?P<username>[a-zA-Z0-9\.\-_]+)/$'
    suffix = 'username'

class RestPermissionGroupIdPermission(RestPermissionGroupId):
    regex = r'^permission/$'
    suffix = 'permission'

class RestPermissionGroupIdPermissionCount(RestPermissionGroupIdPermission):
    regex = r'^permission/count/$'
    suffix = 'count'

class RestPermissionPermission(RestPermission):
    regex = r'^permission/$'
    name = 'permission'

class RestPermissionPermissionEntity(RestPermission):
    regex = r'^(?P<content_type>[a-z\_]+)/(?P<ent_id>[0-9]+)/$'
    name = 'entity'

# Group name validator
GROUP_NAME_VALIDATOR = {"type": "string", "minLength": 3, "maxLength": 80, "pattern": "^[a-zA-Z0-9\-\_]+$"}

# Group name validator optional
GROUP_NAME_VALIDATOR_OPTIONAL = {"type": "string", "minLength": 3, "maxLength": 80, "pattern": "^[a-zA-Z0-9\-\_]+$",
                                 "required": False}

@RestPermissionType.def_request(Method.GET, Format.JSON)
@cache_page(60*60*24, cache='default', key_prefix='collgate-cache')
def get_permission_types(request):
    """
    Get the list of permissions type in JSON
    """
    logger.debug("Cache miss for permission.permission-type")

    ignore_list = [
        'admin.',
        'contenttypes.',
        'guardian.',
        'main.profile.',
        'main.settings.',
        'sessions.',
        'sites.',
    ]

    for module in module_manager.modules:
        if hasattr(module, 'ignored_permission_types'):
            ignore_list.extend(module.ignored_permission_types)

    types = []

    for perm in Permission.objects.all().select_related('content_type'):
        pid = "%s.%s.%s" % (perm.content_type.app_label, perm.content_type.model, perm.codename)
        add = True
        for ignore_pattern in ignore_list:
            if pid.startswith(ignore_pattern):
                add = False
                break

        if add:
            types.append({
                'id': perm.id,
                'value': pid,
                'label': perm.name
            })

    return HttpResponseRest(request, types)

@RestPermissionUser.def_auth_request(Method.GET, Format.JSON, staff=True)
def get_users_list(request):
    results_per_page = int_arg(request.GET.get('more', 30))
    cursor = json.loads(request.GET.get('cursor', 'null'))
    limit = results_per_page
    sort_by = json.loads(request.GET.get('sort_by', '["username"]'))
    order_by = sort_by

    cq = CursorQuery(User)

    if request.GET.get('search'):
        search = json.loads(request.GET['search'])
        cq.filter(search)

    if request.GET.get('filters'):
        filters = json.loads(request.GET['filters'])
        cq.filter(filters)

    cq.cursor(cursor, order_by)
    cq.order_by(order_by).limit(limit)
    cq.set_count('user_permissions')

    user_items = []

    for user in cq:
        user_items.append({
            'id': user.id,
            'username': user.username,
            'first_name': user.first_name,
            'last_name': user.last_name,
            'email': user.email,
            'is_active': user.is_active,
            'is_staff': user.is_staff,
            'is_superuser': user.is_superuser,
            'num_permissions': user.user_permissions.all().count()  # optimized by the set_count
        })

    results = {
        'perms': get_permissions_for(request.user, "auth", "user"),
        'items': user_items,
        'prev': cq.prev_cursor,
        'cursor': cursor,
        'next': cq.next_cursor
    }

    return HttpResponseRest(request, results)

@RestPermissionUserCount.def_auth_request(Method.GET, Format.JSON, staff=True)
def get_users_list_count(request):
    cq = CursorQuery(User)

    if request.GET.get('search'):
        search = json.loads(request.GET['search'])
        cq.filter(search)

    if request.GET.get('filters'):
        filters = json.loads(request.GET['filters'])
        cq.filter(filters)

    results = {
        'perms': [],
        'count': cq.count()
    }

    return HttpResponseRest(request, results)

@RestPermissionGroup.def_auth_request(Method.GET, Format.JSON, staff=True)
def get_groups_list(request):
    results_per_page = int_arg(request.GET.get('more', 30))
    cursor = json.loads(request.GET.get('cursor', 'null'))
    limit = results_per_page
    sort_by = json.loads(request.GET.get('sort_by', '["name"]'))
    order_by = sort_by

    cq = CursorQuery(Group)

    if request.GET.get('search'):
        search = json.loads(request.GET['search'])
        cq.filter(search)

    if request.GET.get('filters'):
        filters = json.loads(request.GET['filters'])
        cq.filter(filters)

    cq.cursor(cursor, order_by)
    cq.order_by(order_by).limit(limit)
    cq.prefetch_related('permissions')
    cq.prefetch_related('user_set')

    # @todo is it prefetch something ?
    prefetch_permissions_for(request.user, cq)

    group_items = []

    for group in cq:
        group_items.append({
            'id': group.id,
            'name': group.name,
            'num_users': group.user_set.all().count(),  # optimized by prefetch_related
            'num_permissions': group.permissions.all().count(),  # optimized by prefetch_related
            'perms': get_permissions_for(request.user, "auth", "group", group),
        })

    results = {
        'perms': get_permissions_for(request.user, "auth", "group"),
        'items': group_items,
        'prev': cq.prev_cursor,
        'cursor': cursor,
        'next': cq.next_cursor
    }

    return HttpResponseRest(request, results)

@RestPermissionGroupCount.def_auth_request(Method.GET, Format.JSON, staff=True)
def get_groups_list_count(request):
    cq = CursorQuery(Group)

    if request.GET.get('search'):
        search = json.loads(request.GET['search'])
        cq.filter(search)

    if request.GET.get('filters'):
        filters = json.loads(request.GET['filters'])
        cq.filter(filters)

    results = {
        'perms': [],
        'count': cq.count()
    }

    return HttpResponseRest(request, results)

@RestPermissionGroup.def_auth_request(Method.POST, Format.JSON, staff=True, content={
    "type": "object",
    "properties": {
        "name": GROUP_NAME_VALIDATOR
    },
})
def add_group(request):
    group_name = request.data['name']

    if Group.objects.filter(name__exact=group_name).exists():
        raise SuspiciousOperation(_("Group name already in usage"))

    group = Group(name=group_name)
    group.save()

    response = {
        'id': group.id,
        'name': group_name,
        'num_users': 0,
        'num_permissions': 0,
        'perms': get_permissions_for(request.user, "auth", "group"),
    }

    return HttpResponseRest(request, response)

@RestPermissionGroupId.def_auth_request(Method.GET, Format.JSON)
def get_group_details(request, grp_id):
    group = get_object_or_404(Group, id=int(grp_id))

    result = {
        'id': group.id,
        'name': group.name,
        'num_users': group.user_set.all().count(),
        'num_permissions': group.permissions.all().count(),
        'perms': get_permissions_for(request.user, "auth", "group", group),
    }

    return HttpResponseRest(request, result)

@RestPermissionGroupId.def_auth_request(Method.DELETE, Format.JSON, staff=True)
def delete_group(request, grp_id):
    group = get_object_or_404(Group, id=int(grp_id))

    if group.user_set.all().count() > 0:
        raise SuspiciousOperation(_("Only empty groups can be deleted"))

    group.delete()

    return HttpResponseRest(request, {})

@RestPermissionGroupId.def_auth_request(
    Method.PATCH, Format.JSON, content={
        "type": "object",
        "properties": {
            "name": GROUP_NAME_VALIDATOR_OPTIONAL
        },
    },
    staff=True)
def patch_group(request, grp_id):
    group = get_object_or_404(Group, id=int(grp_id))
    group_name = request.data.get('name')

    update = False
    result = {'id': group.pk, 'perms': get_permissions_for(request.user, "auth", "group")}

    if group_name and group_name != group.name:
        if Group.objects.filter(name__exact=group_name).exists():
            raise SuspiciousOperation(_("Group name already in usage"))

        group.name = group_name
        group.full_clean()
        result['name'] = group.name

        update = True

    if update:
        group.save()

    return HttpResponseRest(request, result)

@RestPermissionGroupSearch.def_auth_request(Method.GET, Format.JSON, ('filters',))
def search_group(request):
    groups = GroupFilter.search(json.loads(request.GET['filters']))

    items = []

    response = {
        'total_count': groups.count(),
        'page': 1,
        'items': items
    }

    if groups:
        for g in groups:
            items.append({"id": str(g.id), "label": g.name, "value": g.name})

    return HttpResponseRest(request, response)

@RestPermissionUserSearch.def_auth_request(Method.GET, Format.JSON, ('filters',))
def search_user(request):
    users = UserFilter.search(json.loads(request.GET['filters']))

    items = []

    response = {
        'total_count': users.count(),
        'page': 1,
        'items': items
    }

    if users:
        for u in users:
            items.append({"id": str(u.id), "label": "%s (%s)" % (u.get_full_name(), u.username), "value": u.username})

    return HttpResponseRest(request, response)

class Perm:
    def __init__(self, permissions, model, obj, object_name):
        self.permissions = permissions
        self.model = model
        self.object = obj
        self.object_name = object_name

    def __lt__(self, other):
        if self.object_name == other.object_name:
            return self.model < other.model
        else:
            if self.object_name and other.object_name:
                return self.object_name < other.object_name
            elif not self.object_name:
                return self
            else:
                return other.object_name

    def __eq__(self, other):
        return (self.object_name == other.object_name) and (self.model == other.model)

@RestPermissionUserNamePermission.def_auth_request(Method.GET, Format.JSON, staff=True)
def get_user_permissions(request, username):
    user = get_object_or_404(User, username=username)

    permissions = []

    # from classification.models import Taxon
    # content_type = ContentType.objects.get_by_natural_key('classification', 'Taxon')
    # obj = get_object_or_404(Taxon, id=1)
    # UserObjectPermission.objects.assign_perm('change_taxon', user=User.objects.get(username='fscherma'), obj=obj)

    checkout = Permission.objects.filter(user=user).select_related('content_type')
    lookup = {}

    for perm in checkout:
        if perm.content_type.model in lookup:
            perms = lookup[perm.content_type.model]
        else:
            perms = []
            lookup[perm.content_type.model] = perms

        perms.append({
            'id': perm.codename,
            'name': perm.name,
            'app_label': perm.content_type.app_label,
        })

    for k, v in lookup.items():
        permissions.append(Perm(v, k, None, None))

    checkout = UserObjectPermission.objects.filter(user=user).select_related('permission', 'content_type')
    lookup = {}

    for perm in checkout:
        obj_name = perm.content_type.get_object_for_this_type(id=perm.object_pk).name

        if (perm.object_pk, perm.content_type.model, obj_name) in lookup:
            perms = lookup[(perm.object_pk, perm.content_type.model, obj_name)]
        else:
            perms = []
            lookup[(perm.object_pk, perm.content_type.model, obj_name)] = perms

        perms.append({
            'id': perm.permission.codename,
            'name': perm.permission.name,
            'app_label': perm.content_type.app_label,
        })

    for k, v in lookup.items():
        permissions.append(Perm(v, k[1], k[0], k[2]))

    permissions.sort()

    response = {
        'username': user.username,
        'permissions': permissions,
        'perms': get_permissions_for(request.user, "auth", "permission"),
    }

    return HttpResponseRest(request, response)

@RestPermissionUserName.def_admin_request(Method.GET, Format.JSON)
def get_user_info(request, username):
    user = get_object_or_404(User, username=username)

    result = {
        'id': user.pk,
        'username': user.username,
        'first_name': user.first_name,
        'last_name': user.last_name
    }

    return HttpResponseRest(request, result)

@RestPermissionUserName.def_admin_request(Method.PATCH, Format.JSON, content={
    "type": "object",
    "properties": {
        "is_active": {"type": "boolean", "required": False},
        "is_staff": {"type": "boolean", "required": False},
        "is_superuser": {"type": "boolean", "required": False},
    },
})
@transaction.atomic
def patch_user(request, username):
    user = get_object_or_404(User, username=username)
    update = False

    result = {'id': user.pk}

    if 'is_active' in request.data:
        update = True
        user.is_active = request.data['is_active']
        result['is_active'] = user.is_active

        # update the pending state if necessary
        try:
            profile = Profile.objects.get(user=user)
            if user.is_active and profile.pending:
                profile.pending = False
                profile.save()
        except Profile.DoesNotExist:
            pass

    if 'is_staff' in request.data:
        update = True
        user.is_staff = request.data['is_staff']
        result['is_staff'] = user.is_staff

    if 'is_superuser' in request.data:
        update = True
        user.is_superuser = request.data['is_superuser']
        result['is_superuser'] = user.is_superuser

    if update:
        user.save()

    return HttpResponseRest(request, result)

@RestPermissionUserNamePermission.def_auth_request(
    Method.POST, Format.JSON, content={
        "type": "object",
        "properties": {
            "permission": Entity.PERMISSION_VALIDATOR,
            "content_type": Entity.CONTENT_TYPE_VALIDATOR,
            "object": {"type": "integer", 'required': False},
        },
    },
    perms={'auth.change_user': _("You are not allowed to changer a user")},
    staff=True)
def add_user_permission(request, username):
    permission = request.data['permission']
    content_type = request.data['content_type']
    object_id = int(request.data['object']) if 'object' in request.data else None

    if content_type == "auth.permission" and not request.user.is_superuser:
        raise PermissionDenied(_("Only a superuser can change an auth.permission"))

    user = get_object_or_404(User, username=username)

    app_label, model = content_type.split('.')
    # content_type = get_object_or_404(ContentType, app_label=app_label, model=model)
    content_type = ContentType.objects.get_by_natural_key(app_label, model)

    if not object_id:
        perm = get_object_or_404(Permission, codename=permission, content_type=content_type)
        user.user_permissions.add(perm)
    else:
        obj = get_object_or_404(content_type.model, id=object_id)
        UserObjectPermission.objects.assign_perm(permission, user=user, obj=obj)

    return HttpResponseRest(request, {})

@RestPermissionUserNamePermission.def_auth_request(
    Method.PATCH, Format.JSON, content={
        "type": "object",
        "properties": {
            "action": {"type": "string", "pattern": "^(add|remove)$"},
            "target": {"type": "string", "pattern": "^permission$"},
            "permission": Entity.PERMISSION_VALIDATOR,
            "content_type": Entity.CONTENT_TYPE_VALIDATOR,
            "object": {"type": "integer", 'required': False},
        },
    },
    perms={'auth.change_user': _("You are not allowed to changer a user")},
    staff=True)
def delete_user_permission(request, username):
    action = request.data['action']
    permission = request.data['permission']
    content_type = request.data['content_type']
    object_id = int(request.data['object']) if 'object' in request.data else None

    if content_type == "auth.permission" and not request.user.is_superuser:
        raise PermissionDenied(_("Only a superuser can change an auth.permission"))

    user = get_object_or_404(User, username=username)

    app_label, model = content_type.split('.')
    content_type = ContentType.objects.get_by_natural_key(app_label, model)

    if not object_id:
        perm = get_object_or_404(Permission, codename=permission, content_type=content_type)
        if action == "add":
            user.user_permissions.add(perm)
        elif action == "remove":
            user.user_permissions.remove(perm)
        else:
            raise SuspiciousOperation('Invalid patch action')
    else:
        obj = get_object_or_404(content_type.model, id=object_id)

        if action == "add":
            UserObjectPermission.objects.assign_perm(permission, user=user, obj=obj)
        elif action == "remove":
            UserObjectPermission.objects.remove_perm(permission, user=user, obj=obj)
        else:
            raise SuspiciousOperation('Invalid patch action')

    return HttpResponseRest(request, {})

@RestPermissionGroupIdPermission.def_auth_request(Method.GET, Format.JSON, staff=True)
def get_group_permissions(request, grp_id):
    group = get_object_or_404(Group, id=int(grp_id))
    permissions = []

    checkout = Permission.objects.filter(group=group).select_related('content_type')
    lookup = {}

    for perm in checkout:
        if perm.content_type.model in lookup:
            perms = lookup[perm.content_type.model]
        else:
            perms = []
            lookup[perm.content_type.model] = perms

        perms.append({
            'id': perm.codename,
            'name': perm.name,
            'app_label': perm.content_type.app_label,
        })

    for k, v in lookup.items():
        permissions.append(Perm(v, k, None, None))

    checkout = GroupObjectPermission.objects.filter(group=group).select_related('permission', 'content_type')
    lookup = {}

    for perm in checkout:
        obj_name = perm.content_type.get_object_for_this_type(id=perm.object_pk).name

        if (perm.object_pk, perm.content_type.model, obj_name) in lookup:
            perms = lookup[(perm.object_pk, perm.content_type.model, obj_name)]
        else:
            perms = []
            lookup[(perm.object_pk, perm.content_type.model, obj_name)] = perms

        perms.append({
            'id': perm.permission.codename,
            'name': perm.permission.name,
            'app_label': perm.content_type.app_label,
        })

    for k, v in lookup.items():
        permissions.append(Perm(v, k[1], k[0], k[2]))

    permissions.sort()

    response = {
        'id': group.id,
        'name': group.name,
        'permissions': permissions,
        'perms': get_permissions_for(request.user, "auth", "permission"),
    }

    return HttpResponseRest(request, response)

@RestPermissionGroupIdPermission.def_auth_request(
    Method.POST, Format.JSON, content={
        "type": "object",
        "properties": {
            "permission": Entity.PERMISSION_VALIDATOR,
            "content_type": Entity.CONTENT_TYPE_VALIDATOR,
            "object": {"type": "integer", 'required': False},
        },
    },
    perms={'auth.change_group': _("You are not allowed to change a group")},
    staff=True)
def add_group_permission(request, grp_id):
    permission = request.data['permission']
    content_type = request.data['content_type']
    object_id = int(request.data['object']) if 'object' in request.data else None

    if content_type == "auth.permission" and not request.user.is_superuser:
        raise PermissionDenied(_("Only a superuser can change an auth.permission"))

    group = get_object_or_404(Group, id=int(grp_id))

    app_label, model = content_type.split('.')
    content_type = ContentType.objects.get_by_natural_key(app_label, model)

    if not object_id:
        perm = get_object_or_404(Permission, codename=permission, content_type=content_type)
        group.permissions.add(perm)
    else:
        obj = get_object_or_404(content_type.model, id=object_id)
        GroupObjectPermission.objects.assign_perm(permission, group=group, obj=obj)

    return HttpResponseRest(request, {})

@RestPermissionGroupIdPermission.def_auth_request(
    Method.PATCH, Format.JSON, content={
        "type": "object",
        "properties": {
            "action": {"type": "string", "pattern": "^(add|remove)$"},
            "target": {"type": "string", "pattern": "^permission$"},
            "permission": Entity.PERMISSION_VALIDATOR,
            "content_type": Entity.CONTENT_TYPE_VALIDATOR,
            "object": {"type": "integer", 'required': False},
        },
    },
    perms={'auth.change_group': _("You are not allowed to change a group")},
    staff=True)
def change_group_permission(request, grp_id):
    action = request.data['action']
    permission = request.data['permission']
    content_type = request.data['content_type']
    target = request.data['target']
    object_id = int(request.data['object']) if 'object' in request.data else None

    if content_type == "auth.permission" and not request.user.is_superuser:
        raise PermissionDenied(_("Only a superuser can change an auth.permission"))

    group = get_object_or_404(Group, id=int(grp_id))

    app_label, model = content_type.split('.')
    content_type = ContentType.objects.get_by_natural_key(app_label, model)

    if target == "permission":
        if not object_id:
            perm = get_object_or_404(Permission, codename=permission, content_type=content_type)
            if action == "add":
                group.permissions.add(perm)
            elif action == "remove":
                group.permissions.remove(perm)
            else:
                raise SuspiciousOperation('Invalid action')
        else:
            obj = get_object_or_404(content_type.model, id=object_id)

            if action == "add":
                GroupObjectPermission.objects.assign_perm(permission, group=group, obj=obj)
            elif action == "remove":
                GroupObjectPermission.objects.remove_perm(permission, group=group, obj=obj)
            else:
                raise SuspiciousOperation('Invalid action')

    return HttpResponseRest(request, {})

@RestPermissionGroupIdUser.def_auth_request(Method.POST, Format.JSON, content={
        "type": "object",
        "properties": {
            "username": {"type": "string", 'minLength': 3, 'maxLength': 150},
        },
    },
    perms={'auth.change_group': _("You are not allowed to add a user into a group")},
    staff=True)
def group_add_user(request, grp_id):
    group = get_object_or_404(Group, id=int(grp_id))
    user = get_object_or_404(User, username=request.data['username'])

    if user in group.user_set.all():
        raise SuspiciousOperation(_("User already exists into the group"))

    user.groups.add(group)

    response = {
        'id': user.id,
        'username': user.username,
        'first_name': user.first_name,
        'last_name': user.last_name,
        'is_active': user.is_active,
        'is_staff': user.is_staff,
        'is_superuser': user.is_superuser,
        'email': user.email}

    return HttpResponseRest(request, response)

@RestPermissionGroupIdUserName.def_auth_request(
    Method.DELETE, Format.JSON,
    perms={'auth.change_group': _("You are not allowed to remove a user from a group")},
    staff=True)
def group_delete_user(request, grp_id, username):
    group = get_object_or_404(Group, id=int(grp_id))
    user = get_object_or_404(User, username=username)

    if user not in group.user_set.all():
        raise SuspiciousOperation(_("User does not exists into the group"))

    user.groups.remove(group)

    return HttpResponseRest(request, {})

@RestPermissionGroupIdUser.def_auth_request(Method.GET, Format.JSON, staff=True)
def get_users_list_for_group(request, grp_id):
    results_per_page = int_arg(request.GET.get('more', 30))
    cursor = json.loads(request.GET.get('cursor', 'null'))
    limit = results_per_page
    sort_by = json.loads(request.GET.get('sort_by', '["username"]'))
    order_by = sort_by

    group = get_object_or_404(Group, id=int(grp_id))
    cq = CursorQuery(User)

    if request.GET.get('search'):
        search = json.loads(request.GET['search'])
        cq.filter(search)

    if request.GET.get('filters'):
        filters = json.loads(request.GET['filters'])
        cq.filter(filters)

    cq.inner_join(Group, related_name='user_set', group=group.pk)
    cq.cursor(cursor, order_by)
    cq.order_by(order_by).limit(limit)

    user_items = []

    for user in cq:
        user_items.append({
            'id': user.id,
            'username': user.username,
            'first_name': user.first_name,
            'last_name': user.last_name,
            'email': user.email,
            'is_active': user.is_active,
            'is_staff': user.is_staff,
            'is_superuser': user.is_superuser,
        })

    results = {
        'perms': get_permissions_for(request.user, "auth", "group", group),
        'items': user_items,
        'prev': cq.prev_cursor,
        'cursor': cursor,
        'next': cq.next_cursor
    }

    return HttpResponseRest(request, results)

@RestPermissionPermission.def_auth_request(Method.GET, Format.JSON)
def get_self_permissions(request):
    """
    User get its own permissions for its session.
    """
    user = get_object_or_404(User, username=request.user.username)

    permissions = []

    # at user level and groups
    checkout = Permission.objects.filter(
        Q(user=user) | Q(group__in=user.groups.all().values_list('id'))).select_related('content_type')

    for perm in checkout:
        permissions.append({
            'id': perm.codename,
            'name': perm.name,
            'app_label': perm.content_type.app_label,
        })

    return HttpResponseRest(request, permissions)

@RestPermissionPermissionEntity.def_auth_request(Method.GET, Format.JSON)
def get_self_permissions_for_entity(request, content_type, ent_id):
    """
    User get its own permissions for its session for a specific entity
    """
    user = get_object_or_404(User, username=request.user.username)

    permissions = []

    app_label, model = content_type.split('.')
    content_type = ContentType.objects.get_by_natural_key(app_label, model)

    entity = get_object_or_404(content_type.model_class(), int(ent_id))

    perms = get_perms(user, entity)
    permission_details = Permission.objects.find(codename__in=perms)

    lookup = {}

    for perm in permission_details:
        lookup[perm.codename] = perm.name

    for perm in perms:
        perm_detail = lookup.get(perm)

        if perm_detail:
            permissions.append({
                'id': perm,
                'name': perm_detail,
                'app_label': app_label
            })

    return HttpResponseRest(request, permissions)


"""
Project all the temporal networks to the same eigen-system and measure change of the
total variation in that eigen-system
"""

from utils.readFile import readSubjectFiles
from numpy import linalg as LA
import numpy as np

def diff_eig_val(temp_net):
    print("*** Project all the temporal networks to the "
          "same eigen-system and measure change of the "
          "total variation in that eigen-system ***")

    t = len(temp_net)
    baseline = temp_net[0]
    eigval = np.zeros((len(baseline), t))
    eigval[:, 0], eigvec = LA.eigh(baseline)

    for i in range(1, t):
        D = np.diag(temp_net[i].sum(axis=1))
        L = D - temp_net[i]
        projection = np.matmul(np.matmul(eigvec.T, L), eigvec)
        #eigval[:, i] = np.diag(projection)
        eigval[:, i] = np.diag(np.sqrt(np.dot(projection, projection.T)))

    diff = np.diag(np.ones(t - 1) * -1)
    diff = np.vstack((np.ones((1, t - 1)), diff))
    diff_eigval = np.dot(eigval, diff)
    normalizer = eigval[:, 0][:, np.newaxis]
    #diff_eigval /= normalizer
    print(diff_eigval)

def diff_eig_vec(temp_net, eig_num=0):
    print("*** Angular difference between "
          "eigen vectors of the temporal networks ***")

    t = len(temp_net)
    n = len(temp_net[0])
    eigvec = np.zeros((n, t))

    for i in range(0, t):
        D = np.diag(temp_net[i].sum(axis=1))
        L = D - temp_net[i]
        _, eig = LA.eigh(L)
        eigvec[:, i] = eig[:, eig_num]

    ang_dist = np.abs(np.dot(eigvec.T, eigvec))
    np.set_printoptions(formatter={'float': lambda x: "{0:0.2f}".format(x)})
    print(ang_dist)

if __name__ == '__main__':
    # Input
    s_name = '027_S_5109'
    _, temp_net = readSubjectFiles(s_name)
    temp_net, _ = readSubjectFiles(s_name)
    diff_eig_vec(temp_net, 4)
    # diff_eig_val(temp_net)


#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
File  :   helper.py
Author:   dangjinming(jmdang777@qq.com)
Date  :   2022/3/16
Desc  :   common func
"""

import logging
import numpy as np
# logging.basicConfig(format='%(asctime)s-%(levelname)s - %(message)s', level=logging.INFO)
from sklearn.metrics import  precision_score, recall_score, f1_score

def multi_classify_prf_macro(y_true, y_pre):
    """macro prf
    每个类别看作二分类,每个TP FN FP TN,算prf后平均
    """
    p = precision_score(y_true=y_true, y_pred=y_pre, average="macro")
    r = recall_score(y_true=y_true, y_pred=y_pre, average="macro")
    f = f1_score(y_true=y_true, y_pred=y_pre, average="macro")
    return p,r,f

def multi_classify_prf_micro(y_true, y_pre):
    """micro, prf
    每个类别看作二分类,每个TP FN FP TN,先求和后算一次prf
    """
    p = precision_score(y_true=y_true, y_pred=y_pre, average="micro")
    r = recall_score(y_true=y_true, y_pred=y_pre, average="micro")
    f = f1_score(y_true=y_true, y_pred=y_pre, average="micro")
    return p,r,f

def multi_classify_prf_self(y_true, y_pred):
    """
    除了0类其他类别的PRF
    """
    y_true=np.argmax(y_true,1)
    y_pred = np.argmax(y_pred, 1)
    y_preNotNA = [int(i) for i in np.greater(y_pred, 0)]
    pre_num = np.sum(y_preNotNA)
    y_trueNotNA = [int(i) for i in np.greater(y_true, 0)]
    true_num = np.sum(y_trueNotNA)
    equal_num = np.sum(np.multiply([i for i in np.equal(y_true, y_pred)], y_preNotNA))
    try:
        precision = equal_num / pre_num
        recall = equal_num / true_num
        f1 = (2 * precision * recall) / (precision + recall)
    except ZeroDivisionError:
        precision, recall, f1 = 0, 0, 0
    return precision, recall, f1

def extract_entity_crf(pre_label, predict_length, label_encoder=None):
    """ 从预测标签抽取实体,标签个数: label_encoder.size() * 2 - 1
    [IN]  pre_label: 模型预测结果
          predict_length: 预测数据真实长度
          label_encoder: 标签编码
    """
    pre_entity = []
    for label, length in zip(pre_label, predict_length):
        item = []
        if length >= len(label):
            length = len(label)
        index = 0
        while index < length:
            if label[index] != 0:
                start = index
                start_type_id = label[index]
                end = index + 1
                while end < length and label[end]==start_type_id + 1:
                    end += 1
                    index += 1
                if start_type_id // 2 + 1 not in label_encoder.id_label_dict:
                    break
                item.append([start - 1, end - 1, label_encoder.inverse_transform(start_type_id // 2 + 1)])
            index += 1
        pre_entity.append(item)
    return pre_entity

def extract_entity_crf_BIO(pre_label, predict_length):
    """ 从预测标签抽取实体,只有三种标签   BIO
    [IN]  pre_label: 模型预测结果
          predict_length: 预测数据真实长度
          label_encoder: 标签编码
    """
    pre_entity = []
    for label, length in zip(pre_label, predict_length):
        item = []
        if length >= len(label):
            length = len(label)
        index = 0
        while index < length:
            if label[index] == 1:
                start = index
                end = index + 1
                while end < length and label[end]==2:
                    end += 1
                item.append([start - 1, end - 1])
            index += 1
        pre_entity.append(item)
    return pre_entity

def extract_entity_dp(pre_label, predict_length, label_encoder):
    """ 从预测标签抽取实体
    [IN]  pre_label: 模型预测结果
          predict_length: 预测数据真实长度
          label_encoder: 标签编码
    """
    pre_entity = []
    for label, length in zip(pre_label, predict_length):
        item = []
        if length >= len(label[0]):
            length = len(label[0])
        for start in range(length):
            if (label[0][start] != 0):
                for end in range(start, length):
                    if (label[1][end] != 0):
                        item.append([start - 1, end, label_encoder.inverse_transform(label[1][end])])
                        break
        pre_entity.append(item)
    return pre_entity

if __name__ == '__main__':
    print(extract_entity_crf([[1, 2, 2, 2, 1, 2, 2]], [7]))


import os
from zipfile import ZipFile

def create_assets_archive(directory):
    assets_name=directory.replace("assets-", "").split(".")[-1]
    print(f"Creating Assets.{assets_name}.zip for folder {directory}")
    with ZipFile(f'Assets.{assets_name}.zip', 'w') as myzip:
        dir_leaf = ""
        for root, _, files in os.walk(directory, topdown=False):
            for fi in files:
                print(f"Adding bin {fi} to Assets.{assets_name}.zip")
                myzip.write(os.path.join(root, fi), arcname=f"assets/{fi}")
                dir_leaf = os.path.join(root, fi)
        asset_path = directory.replace("assets-", "").replace(".", "/")
        for root, _, files in os.walk(os.path.join("Assets", asset_path, "fonts"), topdown=False):
            for fi in files:
                print(f"Adding mks_font/{fi} to Assets.{assets_name}.zip")
                myzip.write(os.path.join(root, fi), arcname=f"assets/{fi}")

def create_firmware_archive(directory):
    firmware_name = directory.replace("firmware-", "")
    print(f"Creating {firmware_name}.zip for folder {directory}")
    with ZipFile(f'{firmware_name}.zip', 'w') as myzip:
        myzip.write(f"{directory}/Robin_nano35.bin", arcname="Robin_nano35.bin")

for directory in os.listdir("."):
    if directory.startswith("assets-"):
        create_assets_archive(directory)
    elif directory.startswith("firmware-"):
        create_firmware_archive(directory)


"""
Imgur CLI
"""

import argparse
import logging
import os
import traceback
import sys

from collections import namedtuple

import imgurpython

from imgur_cli import __version__
from imgur_cli import cli_api
from imgur_cli import exceptions
from imgur_cli.utils import cli_arg

logger = logging.getLogger(__name__)

def imgur_credentials():
    ImgurCredentials = namedtuple('ImgurCredentials',
                                  ['client_id', 'client_secret', 'access_token',
                                   'refresh_token', 'mashape_key'])

    client_id = os.environ.get('IMGUR_CLIENT_ID')
    client_secret = os.environ.get('IMGUR_CLIENT_SECRET')
    access_token = os.environ.get('IMGUR_ACCESS_TOKEN')
    refresh_token = os.environ.get('IMGUR_REFRESH_TOKEN')
    mashape_key = os.environ.get('IMGUR_MASHAPE_KEY')

    if not client_id or not client_secret:
        raise imgurpython.client.ImgurClientError('Client credentials not found. '
                                                  'Ensure you have both client id '
                                                  'and client secret')

    return ImgurCredentials(client_id, client_secret, access_token,
                            refresh_token, mashape_key)

class ImgurCli:

    def __init__(self):
        self.subcommands = {}
        self.parser = None
        self.client = None
        self.subparsers = {}

    @property
    def base_parser(self):
        parser = argparse.ArgumentParser(prog='imgur', description=__doc__.strip(),
                                         epilog='See "imgur help <SUBPARSER> '
                                         '<SUBCOMMAND>" for help on a specific '
                                         'subparser or subcommand')

        # Global arguments
        parser.add_argument('-v', '--version', action='version',
                            version='%(prog)s {0}'.format(__version__))

        return parser

    def generate_parser(self):
        self.parser = self.base_parser
        base_subparser = self.parser.add_subparsers(metavar='<subparsers>')
        self._add_help(self, base_subparser)
        self._add_subparsers(base_subparser)
        self._add_subcommands(cli_api)

    def _generate_subcommand(self, name, callback, subparser):
        description = callback.__doc__ or ''
        action_help = description.strip()
        arguments = getattr(callback, 'arguments', [])
        subcommand = subparser.add_parser(name, help=action_help,
                                          description=description,
                                          add_help=False)
        subcommand.add_argument('-h', '--help', action='help',
                                help=argparse.SUPPRESS)
        for args, kwargs in arguments:
            subcommand.add_argument(*args, **kwargs)
        subcommand.set_defaults(func=callback)
        self.subcommands[name] = subcommand

    def _add_help(self, cli_api, subparser):
        callback = getattr(self, 'cmd_help')
        self._generate_subcommand('help', callback, subparser)

    def _add_subparsers(self, subparser):
        for name, description in cli_api.SUBPARSERS.items():
            parser = subparser.add_parser(name, help=description,
                                          description=description,
                                          add_help=False)
            parser.add_argument('-h', '--help', action='help',
                                help=argparse.SUPPRESS)
            self.subparsers[name] = {}
            self.subparsers[name]['parser'] = parser
            self.subparsers[name]['subparser'] = (
                parser.add_subparsers(metavar='<subcommands>')
            )

    def _add_subcommands(self, cli_api):
        for attr in (action for action in dir(cli_api)
                     if action.startswith('cmd_')):
            callback = getattr(cli_api, attr)
            subparser_name = getattr(callback, 'subparser', None)
            name = (attr[len('cmd_{0}_'.format(subparser_name.replace('-', '_'))):]
                    .replace('_', '-'))
            subparser = self.subparsers[subparser_name]['subparser']
            self._generate_subcommand(name, callback, subparser)

    @cli_arg('subparser', metavar='<subparser>', nargs='?',
             help='Display help for <subparser>')
    @cli_arg('subcommand', metavar='<subcommand>', nargs='?',
             help='Display help for <subcommand>')
    def cmd_help(self, args):
        """Display help about this program or one of its subparsers"""
        if args.subparser:
            if args.subcommand:
                try:
                    self.subcommands[args.subcommand].print_help()
                except KeyError:
                    raise exceptions.CommandError('{0} is not valid subcommand'
                                                  .format(args.subcommand))
            else:
                try:
                    self.subparsers[args.subparser]['parser'].print_help()
                except KeyError:
                    raise exceptions.CommandError('{0} is not valid subparser'
                                                  .format(args.subparser))
        else:
            self.parser.print_help()

    def main(self, argv):
        self.generate_parser()
        if not argv:
            self.parser.print_help()
            return 0
        args = self.parser.parse_args(argv)
        try:
            # Short-circuit and deal with help right away
            if args.func == self.cmd_help:
                self.cmd_help(args)
                return 0
            credentials = imgur_credentials()
            self.client = imgurpython.ImgurClient(*credentials)
            args.func(self.client, args)
        except AttributeError:
            self.subparsers[argv[0]]['parser'].print_help()

def main():
    try:
        imgur_cli = ImgurCli()
        imgur_cli.main(sys.argv[1:])
    except imgurpython.client.ImgurClientError as e:
        print(e, file=sys.stderr)
        sys.exit(1)
    except Exception as e:
        print(traceback.format_exc())
        sys.exit(1)

if __name__ == '__main__':
    main()