import jsonpickle

from discovered_hash_func_key import DiscoveredHashFuncKey
from analyses.hash_func_patching.hash_func_patching_analysis import HashFuncPatchingAnalysisResults
from analyses.hash_func_discovery.hash_func_discovery_analysis import DiscoveredFunc

def map_md5_to_file_names(results_list):
    md5_to_file_names = {}
    for results in results_list:
        analysis_results = results.analysis_results
        md5 = results.analysis_target.file_md5
        file_name = results.analysis_target.full_file_path
        if md5 not in md5_to_file_names:
            md5_to_file_names[md5] = set()
        md5_to_file_names[md5].add(file_name)
    return md5_to_file_names

def map_md5_to_analysis_errs(results_list):
    md5_to_errs = {}
    for results in results_list:
        analysis_results = results.analysis_results
        all_errs = get_all_errs(analysis_results)
        if len(all_errs) > 0:
            md5 = results.analysis_target.file_md5
            assert md5 not in md5_to_errs
            md5_to_errs[md5] = all_errs
    return md5_to_errs

def map_analysis_err_to_md5s(md5_to_errs):
    err_to_md5s = {}
    for md5, errs in md5_to_errs.items():
        for err in errs:
            if err not in err_to_md5s:
                err_to_md5s[err] = set()
            err_to_md5s[err].add(md5)
    return err_to_md5s

def map_dhf_key_to_dhf(results_list):
    dhf_key_to_dhf = {}
    for results in results_list:
        analysis_results = results.analysis_results
        analysis_target = results.analysis_target
        hash_func_discovery_analysis_results = get_hash_func_discovery_analysis_results(analysis_results)
        discovered_hash_funcs = hash_func_discovery_analysis_results.discovered_hash_funcs
        for discovered_hash_func in discovered_hash_funcs:
            dhf_key = DiscoveredHashFuncKey(analysis_target, discovered_hash_func)
            assert dhf_key not in dhf_key_to_dhf, "We have duplicate keys: {}".format(dhf_key)
            dhf_key_to_dhf[dhf_key] = discovered_hash_func
    return dhf_key_to_dhf

def discard_results_with_analysis_errs(results_list):
    new_results_list = []
    for results in results_list:
        analysis_results = results.analysis_results
        all_errs = get_all_errs(analysis_results)
        if len(all_errs) == 0:
            new_results_list.append(results)
    return new_results_list

def extract_patch_target_dhf_keys(results_list):
    patch_target_dhf_keys = set()
    for results in results_list:
        analysis_results = results.analysis_results
        analysis_target = results.analysis_target

        hash_funcs_to_patch = analysis_results.hash_funcs_to_patch
        if hash_funcs_to_patch is None:
            continue
        for hash_func_to_patch in hash_funcs_to_patch:
            assert type(hash_func_to_patch).__name__ == "DiscoveredFunc"
            dhf_key = DiscoveredHashFuncKey(analysis_target, hash_func_to_patch)
            patch_target_dhf_keys.add(dhf_key)

    return patch_target_dhf_keys

def extract_confirmed_hash_function_dhf_keys(dhf_key_to_patch_details, md5_to_errs):
    confirmed_hash_function_dhf_keys = set()
    nonconfirmed_hash_function_sig_dhf_keys = set()
    nonconfirmed_hash_function_io_dhf_keys = set()
    nonconfirmed_hash_function_err_dhf_keys = set()
    nonconfirmed_hash_function_timeout_dhf_keys = set()
    for dhf_key, patch_details in dhf_key_to_patch_details.items():
        orig_hash_symb_ret_irregularities = patch_details.orig_hash_symb_ret_irregularities
        orig_hash_val_irregularities = patch_details.orig_hash_val_irregularities
        patch_errs = patch_details.errs
        analysis_errs = md5_to_errs[dhf_key.file_md5] if dhf_key.file_md5 in md5_to_errs else []
        milestone_timestamps = patch_details.milestone_timestamps
        last_milestone_label = milestone_timestamps[-1][0]

        #if len(analysis_errs) > 0:
        #    continue

        # We confirm that we do not have symbolic returns
        # We confirm that the input/output relationships match
        if (orig_hash_symb_ret_irregularities is not None and len(orig_hash_symb_ret_irregularities) == 0) and \
            (orig_hash_val_irregularities is not None and len(orig_hash_val_irregularities) == 0):
            assert dhf_key not in confirmed_hash_function_dhf_keys, "We have duplicate keys: {}".format(dhf_key)
            if len(patch_errs) > 0:
                assert "Could not find a signature that works (aborting patching)." not in patch_errs, "There were no hash value irregularities, but we could not find a valid signature?"
            confirmed_hash_function_dhf_keys.add(dhf_key)
            continue

        if len(patch_errs) > 0:
            if "Could not find a signature that works (aborting patching)." in patch_errs:
                assert dhf_key not in nonconfirmed_hash_function_sig_dhf_keys, "We have duplicate keys: {}".format(dhf_key)
                nonconfirmed_hash_function_sig_dhf_keys.add(dhf_key)
                continue

        if (orig_hash_symb_ret_irregularities is not None and len(orig_hash_symb_ret_irregularities) > 0) or (orig_hash_val_irregularities is not None and len(orig_hash_val_irregularities) > 0):
            assert dhf_key not in nonconfirmed_hash_function_io_dhf_keys, "We have duplicate keys: {}".format(dhf_key)

            nonconfirmed_hash_function_io_dhf_keys.add(dhf_key)
            continue

        if any('timeout' in err.lower() for err in analysis_errs):
            if last_milestone_label != 'completed: patch evaluation finished':
                assert dhf_key not in nonconfirmed_hash_function_timeout_dhf_keys, "We have duplicate keys: {}".format(dhf_key)
                nonconfirmed_hash_function_timeout_dhf_keys.add(dhf_key)
                continue

        if len(analysis_errs) > 0:
            assert dhf_key not in nonconfirmed_hash_function_err_dhf_keys, "We have duplicate keys: {}".format(dhf_key)
            nonconfirmed_hash_function_err_dhf_keys.add(dhf_key)
            continue

        raise Exception("Unclassified nonconfirmed hash function: {}".format(dhf_key))

    return confirmed_hash_function_dhf_keys, nonconfirmed_hash_function_sig_dhf_keys, nonconfirmed_hash_function_io_dhf_keys, nonconfirmed_hash_function_timeout_dhf_keys, nonconfirmed_hash_function_err_dhf_keys

def extract_analysis_time_info(results_list):
    min_time, max_time, total_time, ave_time = float('inf'), -float('inf'), 0.0, 0.0
    min_time_file, min_time_md5, max_time_file, max_time_md5 = None, None, None, None
    num = 0
    for results in results_list:
        analysis_target = results.analysis_target
        file_path = analysis_target.full_file_path
        file_md5 = analysis_target.file_md5
        analysis_results = results.analysis_results
        start_time = analysis_results.start_time
        end_time = analysis_results.end_time
        run_time = end_time - start_time
        total_seconds = run_time.total_seconds()
        if total_seconds < min_time:
            min_time = total_seconds
            min_time_file = file_path
            min_time_md5 = file_md5
        if total_seconds > max_time:
            max_time = total_seconds
            max_time_file = file_path
            max_time_md5 = file_md5
        total_time += total_seconds
        num += 1

    ave_time = total_time / num

    return min_time, min_time_file, min_time_md5, max_time, max_time_file, max_time_md5, total_time, ave_time

def map_md5_to_hash_alg_to_dhfs(dhf_key_to_dhf, hash_algs):
    md5_to_dhf_key_to_dhf = group_by_md5(dhf_key_to_dhf)
    md5_to_hash_alg_to_dhfs = {}
    for md5, dhf_key_to_dhf in md5_to_dhf_key_to_dhf.items():
        hash_alg_to_dhf_key_to_dhf = group_by_hash_alg(dhf_key_to_dhf, hash_algs)
        assert md5 not in md5_to_hash_alg_to_dhfs
        md5_to_hash_alg_to_dhfs[md5] = hash_alg_to_dhf_key_to_dhf
    return md5_to_hash_alg_to_dhfs

def extract_lone_dhfs(md5_to_hash_alg_to_dhfs):
    lone_dhf_key_to_dhf = {}
    for md5, hash_alg_to_dhfs in md5_to_hash_alg_to_dhfs.items():
        for hash_alg, dhf_key_to_dhf in hash_alg_to_dhfs.items():
            # There is only one discovered hash func of this hash alg, therefore it is a lone hash func
            if len(dhf_key_to_dhf) == 1:
                dhf_key, dhf = list(dhf_key_to_dhf.items())[0]
                assert dhf_key not in lone_dhf_key_to_dhf
                lone_dhf_key_to_dhf[dhf_key] = dhf
    return lone_dhf_key_to_dhf

# Map the number of hash functions in a binary to the number of binaries with this number of hash functions
def map_num_dhfs_to_num_md5s(dhf_key_to_dhf):
    md5_to_dhf_key_to_dhf = group_by_md5(dhf_key_to_dhf)
    num_dhfs_to_num_md5s = {}

    for md5, dhf_key_to_dhf in md5_to_dhf_key_to_dhf.items():
        num_dhfs = len(dhf_key_to_dhf)
        if num_dhfs not in num_dhfs_to_num_md5s:
            num_dhfs_to_num_md5s[num_dhfs] = 0
        num_dhfs_to_num_md5s[num_dhfs] += 1
    return num_dhfs_to_num_md5s

def group_by_hash_alg(dhf_key_to_something, hash_algs):
    hash_alg_to_something = {hash_alg:{} for hash_alg in hash_algs}
    for dhf_key, something in dhf_key_to_something.items():
        hash_alg = dhf_key.hash_alg
        if hash_alg in hash_alg_to_something:
            assert dhf_key not in hash_alg_to_something[hash_alg]
            hash_alg_to_something[hash_alg][dhf_key] = something
    return hash_alg_to_something

def group_by_md5(dhf_key_to_something):
    md5_to_something = {}
    for dhf_key, something in dhf_key_to_something.items():
        md5 = dhf_key.file_md5
        if md5 not in md5_to_something:
            md5_to_something[md5] = {}
        assert dhf_key not in md5_to_something[md5]
        md5_to_something[md5][dhf_key] = something
    return md5_to_something

def discard_dhf_keys_of_errored_analyses(dhf_key_map, md5_to_errs):
    selcted_dhf_key_map = dhf_key_map.copy()
    for dhf_key in dhf_key_map:
        md5 = dhf_key.file_md5
        if md5 in md5_to_errs:
            selcted_dhf_key_map.pop(dhf_key)

    return selcted_dhf_key_map

def read_results(results_file_name):
    results_list = []
    deserialization_errs = 0
    with open(results_file_name, 'r') as fd:
        for i, line in enumerate(fd):
            try:
                results = jsonpickle.decode(line)
                results_list.append(results)
            except Exception as e:
                deserialization_errs += 1
    return results_list, deserialization_errs

def map_analysis_target_md5_to_results(all_results):
    analysis_target_md5_to_results = {}
    for results in all_results:
        analysis_target = results.analysis_target
        md5 = analysis_target.file_md5
        assert md5 not in analysis_target_md5_to_results, "We have duplicate MD5s: {}".format(md5)
        analysis_target_md5_to_results[md5] = results
    return analysis_target_md5_to_results

def get_results_with_analysis_errs(dhf_key_to_results):
    dhf_with_analysis_errs = {}
    for dhf_key, results_list in dhf_key_to_results.items():
        for analysis_target, analysis_results in results_list:
            all_errs = get_all_errs(analysis_results)
            if len(all_errs) > 0:
                dhf_with_analysis_errs[dhf_key] = results_list
                break
    return dhf_with_analysis_errs

def get_all_errs(analysis_results):
    hash_func_discovery_analysis_results = get_hash_func_discovery_analysis_results(analysis_results)
    return analysis_results.errs + hash_func_discovery_analysis_results.errs

def map_err_to_dhf(dhf_to_results):
    err_to_dhf = {}
    for dhf, results_list in dhf_to_results.items():
        for analysis_target, analysis_results in results_list:
            errs = get_all_errs(analysis_results)
            for err in errs:
                if err not in err_to_dhf:
                    err_to_dhf[err] = set()
                err_to_dhf[err].add(dhf)

    return err_to_dhf

def map_dhf_key_to_patch_details(results_list):
    dhf_key_to_patch_details = {}
    for results in results_list:
        analysis_target = results.analysis_target
        analysis_results = results.analysis_results
        assert type(analysis_results).__name__ == "HashFuncPatchingAnalysisResults", "Not patch analysis results!"
        for hash_func_patch_details in analysis_results.hash_func_patch_details_list:
            discovered_hash_func = hash_func_patch_details.patch_target
            dhf_key = DiscoveredHashFuncKey(analysis_target, discovered_hash_func)
            assert dhf_key not in dhf_key_to_patch_details, "We already have patch details for {}".format(dhf_key)
            dhf_key_to_patch_details[dhf_key] = hash_func_patch_details
    return dhf_key_to_patch_details

def classify_patch_details(dhf_key_to_patch_details, md5_to_errs):
    dhf_key_to_patched = {}
    dhf_key_to_confirmed = {}
    dhf_key_to_timeouts = {}
    dhf_key_to_exceptions = {}
    #dhf_key_to_errs = {}
    dhf_key_to_sig_errs = {}
    dhf_key_to_space_errs = {}
    dhf_key_to_io_irrs = {}
    dhf_key_to_mem_irrs = {}
    dhf_key_to_patch_irrs = {}

    for dhf_key, patch_details in dhf_key_to_patch_details.items():
        dhf = patch_details.patch_target
        analysis_errs = md5_to_errs[dhf_key.file_md5] if dhf_key.file_md5 in md5_to_errs else []
        orig_hash_symb_ret_irregularities = patch_details.orig_hash_symb_ret_irregularities
        orig_hash_val_irregularities = patch_details.orig_hash_val_irregularities
        orig_hash_ext_mem_ref_irregularities = patch_details.orig_hash_ext_mem_ref_irregularities
        patch_hash_ext_mem_ref_irregularities = patch_details.patch_hash_ext_mem_ref_irregularities
        range_irregularities = patch_details.range_irregularities
        patch_errs = patch_details.errs
        milestone_timestamps = patch_details.milestone_timestamps
        last_milestone_label = milestone_timestamps[-1][0]

        if (orig_hash_symb_ret_irregularities is not None and len(orig_hash_symb_ret_irregularities) == 0) and \
                (orig_hash_val_irregularities is not None and len(orig_hash_val_irregularities) == 0):
            assert len(patch_errs) == 0 or "Could not find a signature that works (aborting patching)." not in patch_errs
            assert not ((orig_hash_symb_ret_irregularities is not None and len(orig_hash_symb_ret_irregularities) > 0) or (orig_hash_val_irregularities is not None and len(orig_hash_val_irregularities) > 0))

            dhf_key_to_confirmed[dhf_key] = dhf

        # Successful patches
        if (orig_hash_symb_ret_irregularities is not None and len(orig_hash_symb_ret_irregularities) == 0) and \
                (orig_hash_val_irregularities is not None and len(orig_hash_val_irregularities) == 0) and \
                (orig_hash_ext_mem_ref_irregularities is not None and len(orig_hash_ext_mem_ref_irregularities) == 0) and \
                (patch_hash_ext_mem_ref_irregularities is not None and len(patch_hash_ext_mem_ref_irregularities) == 0) and \
                (range_irregularities is not None and len(range_irregularities) == 0) and \
                last_milestone_label == 'completed: patch evaluation finished':
            assert dhf_key not in dhf_key_to_patched, "We have duplicate keys: {}".format(dhf_key)
            dhf_key_to_patched[dhf_key] = dhf
            continue

        if len(patch_errs) > 0:
            if "Could not find a signature that works (aborting patching)." in patch_errs:
                assert dhf_key not in dhf_key_to_sig_errs, "We have duplicate keys: {}".format(dhf_key)
                dhf_key_to_sig_errs[dhf_key] = dhf
                continue

        if (orig_hash_symb_ret_irregularities is not None and len(orig_hash_symb_ret_irregularities) > 0) or (orig_hash_val_irregularities is not None and len(orig_hash_val_irregularities) > 0):
            assert dhf_key not in dhf_key_to_io_irrs, "We have duplicate keys: {}".format(dhf_key)
            dhf_key_to_io_irrs[dhf_key] = dhf
            continue

        if (orig_hash_ext_mem_ref_irregularities is not None and len(orig_hash_ext_mem_ref_irregularities) > 0):
            assert dhf_key not in dhf_key_to_mem_irrs, "We have duplicate keys: {}".format(dhf_key)
            dhf_key_to_mem_irrs[dhf_key] = dhf
            continue

        if len(patch_errs) > 0:
            if "Could not add replacement hash function to binary (aborting patching)." in patch_errs:
                assert dhf_key not in dhf_key_to_space_errs, "We have duplicate keys: {}".format(dhf_key)
                dhf_key_to_space_errs[dhf_key] = dhf
            else:
                raise Exception("Unhandled patch errs: {}".format(patch_errs))
            continue

        if (patch_hash_ext_mem_ref_irregularities is not None and len(patch_hash_ext_mem_ref_irregularities) > 0) or \
            (range_irregularities is not None and len(range_irregularities) > 0):
            assert dhf_key not in dhf_key_to_patch_irrs, "We have duplicate keys: {}".format(dhf_key)
            dhf_key_to_patch_irrs[dhf_key] = dhf
            continue

        if any('timeout' in err.lower() for err in analysis_errs):
            if last_milestone_label != 'completed: patch evaluation finished':
                assert dhf_key not in dhf_key_to_timeouts, "We have duplicate keys: {}".format(dhf_key)
                dhf_key_to_timeouts[dhf_key] = dhf
                continue

        if len(analysis_errs) > 0:
            assert dhf_key not in dhf_key_to_exceptions, "We have duplicate keys: {}".format(dhf_key)
            dhf_key_to_exceptions[dhf_key] = dhf
            continue

        raise Exception("Unclassified patch details: {}".format(dhf_key))
    assert set(dhf_key_to_patched) | \
        set(dhf_key_to_timeouts) | \
        set(dhf_key_to_exceptions) | \
        set(dhf_key_to_sig_errs) | \
        set(dhf_key_to_space_errs) | \
        set(dhf_key_to_io_irrs) | \
        set(dhf_key_to_mem_irrs) | \
        set(dhf_key_to_patch_irrs) == set(dhf_key_to_patch_details)
    return dhf_key_to_patched, dhf_key_to_confirmed, dhf_key_to_timeouts, dhf_key_to_exceptions, dhf_key_to_sig_errs, dhf_key_to_space_errs, dhf_key_to_io_irrs, dhf_key_to_mem_irrs, dhf_key_to_patch_irrs

def get_discovered_hash_funcs(analysis_results):
    hash_func_discovery_analysis_results = get_hash_func_discovery_analysis_results(analysis_results)
    return hash_func_discovery_analysis_results.discovered_hash_funcs

def get_hash_func_discovery_analysis_results(analysis_results):
    if type(analysis_results).__name__ == "HashFuncPatchingAnalysisResults":
        hash_func_discovery_analysis_results =  analysis_results.hash_funcs_discovery_analysis_results
    elif type(analysis_results).__name__ == "HashFuncDiscoveryAnalysisResults":
        hash_func_discovery_analysis_results = analysis_results
    else:
        raise Exception("I don't know where to get the HashFuncDiscoveryAnalysisResults in type: {}: {}".format(type(analysis_results), analysis_results))
    return hash_func_discovery_analysis_results


"""Automated Evaluation."""

import argparse
from pathlib import Path
from models.run_torch_model import run_torch_cve_model_bert
import pandas as pd

def _create_argument_parser():
    parser = argparse.ArgumentParser(
        description="Evaluate a Pytorch .bin model on historical data."
    )
    parser.add_argument(
        "--data-path",
        metavar="dpath",
        type=str,
        help="the path to the entire historical data.",
        required=True,
    )
    parser.add_argument(
        "--model-path",
        metavar="mpath",
        type=str,
        help="the path to the pytorch model files.",
        required=True,
    )
    parser.add_argument(
        "--excel-path",
        metavar="xlpath",
        type=str,
        help="the path to the directory containing the triage excel files with the final result.",
        required=True,
    )

    return parser

def _main():
    parser = _create_argument_parser()
    args = parser.parse_args()
    ground_truth = _get_triaged_csv_data(args)
    print("Ground truth size: {}".format(ground_truth["url"].unique().shape[0]))
    security_issues = _get_all_security_cves(args)
    triaged_csv = _do_triage(security_issues, args)
    print("Number of issues marked as csv: {}".format(triaged_csv["url"].unique().shape[0]))
    found = triaged_csv[triaged_csv["url"].isin(ground_truth["url"].unique())]
    print("Number of true positives detected by model: {}".format(found["url"].unique().shape[0]))

def _sanitize_column_names(df: pd.DataFrame) -> pd.DataFrame:
    return df.rename(columns={"Feeedback": "Feedback"})

def _get_triaged_csv_data(args) -> pd.DataFrame:
    path = Path(args.excel_path)
    triaged_openshift = pd.read_excel(
        path.joinpath("OpenShift Probable vulnerabilities.xlsx"), sheet_name=None
    )
    triaged_openshift_combined = pd.concat(
        (_sanitize_column_names(v) for k, v in triaged_openshift.items()), sort=False
    ).reset_index(drop=True)
    triaged_kubevirt = pd.read_excel(
        path.joinpath("Kubevirt Probable vulnerabilities.xlsx"), sheet_name=None
    )
    triaged_kubevirt_combined = pd.concat(
        (_sanitize_column_names(v) for k, v in triaged_kubevirt.items()), sort=False
    ).reset_index(drop=True)
    triaged_knative = pd.read_excel(
        path.joinpath("Knative Probable vulnerabilities.xlsx"), sheet_name=None
    )
    triaged_knative_combined = pd.concat(
        (_sanitize_column_names(v) for k, v in triaged_knative.items()), sort=False
    ).reset_index(drop=True)
    triaged_openshift_combined["ecosystem"] = "openshift"
    triaged_knative_combined["ecosystem"] = "knative"
    triaged_kubevirt_combined["ecosystem"] = "kubevirt"
    all_csv = pd.concat(
        [triaged_openshift_combined, triaged_knative_combined, triaged_kubevirt_combined], sort=True
    ).reset_index(drop=True)
    all_csv = all_csv[
        ~all_csv["triage_feedback_comments"].isna()
        | ~all_csv["Feedback"].isna()
        | ~all_csv["triage_is_cve"].isna()
    ]
    return _get_cve_rows(all_csv)

def _get_cve_rows(all_csv: pd.DataFrame) -> pd.DataFrame:
    cve_rows = all_csv[
        ~all_csv["triage_is_cve"].isna()
        | all_csv["Feedback"].str.strip().str.lower().str.contains("yes")
        | all_csv["triage_feedback_comments"].str.strip().str.lower().str.contains("yes")
    ]
    # Inspect this for debugging:
    # non_cve_rows = all_csv[~all_csv.index.isin(cve_rows.index)]
    return cve_rows

def _get_all_security_cves(args) -> pd.DataFrame:
    data_path = Path(args.data_path)
    security_csv_list = list(data_path.glob("**/*probable_security_and_cves*.csv"))
    security_issue_df = pd.concat(
        (pd.read_csv(s) for s in security_csv_list), sort=True
    ).reset_index(drop=True)
    security_issue_df["cve_model_flag"] = 0
    return security_issue_df

def _do_triage(data_frame, args):
    return run_torch_cve_model_bert(data_frame, args.model_path,
                                    batch_size_prediction=8)

if __name__ == "__main__":
    _main()


import argparse
import datetime
import inspect
import importlib.util
import logging
import os
from typing import List
from types import ModuleType

from testcase import TestCase

def import_file(file_path: str):
    """
    Import the python file and return the module object
    :param file_path: full path of the python file to import
    :return: module object
    """
    module_name = inspect.getmodulename(file_path)
    spec = importlib.util.spec_from_file_location(module_name, file_path)
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    return module

class TestCaseFile(object):
    def __init__(self, file_name: str) -> None:
        self._special_test_cases = ["test_module_setup", "test_module_cleanup",
                                    "test_case_setup", "test_case_cleanup"]
        self.file_name: str = file_name
        self.module: ModuleType = import_file(file_name)
        self.test_case_list: List[TestCase] = self._load_test_cases()
        # filled by run.py when it loads the framework
        self.framework_module_setup_output = None
        self.framework_case_setup_tc: TestCase = None
        self.framework_case_cleanup_tc: TestCase = None
        # store this for convenience as its accessed by test case
        self.test_module_setup_tc: TestCase = None
        # cli argument parser (added by run.py during the run)
        self.arg_parser = None
        self.args: argparse.Namespace = argparse.Namespace()
        self.log_dir = ""
        self.start_time: datetime.datetime = None
        self.end_time: datetime.datetime = None
        self.duration: int = 0
        self.logger = logging.getLogger("runner")

    def _load_test_cases(self) -> List[TestCase]:
        func_list = inspect.getmembers(self.module, inspect.isfunction)
        test_case_list: List[TestCase] = []
        for func_name, func_obj in func_list:
            # get all the functions that start with test_
            if not func_name.startswith('test_'):
                continue
            testcase_obj = TestCase(func_obj)
            test_case_list.append(testcase_obj)
        return test_case_list

    def _find_test_case(self, test_case_name: str) -> TestCase:
        matched_tc = None
        for tc in self.test_case_list:
            if tc.name == test_case_name:
                matched_tc = tc
                break
        return matched_tc

    def get_test_cases(self, special=False) -> List[TestCase]:
        if special == True:
            return self.test_case_list
        test_cases = []
        for tc in self.test_case_list:
            if tc.name in self._special_test_cases:
                continue
            test_cases.append(tc)
        return test_cases

    def run_test_module_setup(self):
        self.test_module_setup_tc = self._find_test_case("test_module_setup")
        if not self.test_module_setup_tc:
            return
        self.logger.info(
            f"--Running test_module_setup from {self.file_name}")
        # pass the output of the framework module setup into this
        self.test_module_setup_tc.framework_module_setup_output = self.framework_module_setup_output
        self.test_module_setup_tc.run(self.log_dir)
        self.logger.info(
            f"--Completed test_module_setup from {self.file_name}")

    def run_test_case(self, tc: TestCase):
        # let the test case know about the other init/cleanup tests that it need to run
        tc.framework_case_setup_tc = self.framework_case_setup_tc
        tc.framework_case_cleanup_tc = self.framework_case_cleanup_tc
        tc.test_case_setup_tc = self._find_test_case("test_case_setup")
        tc.test_case_cleanup_tc = self._find_test_case("test_case_cleanup")
        # pass the outputs from the module setups to the test case
        tc.framework_module_setup_output = self.framework_module_setup_output
        if self.test_module_setup_tc:
            tc.test_module_setup_output = self.test_module_setup_tc.output
        tc.args = self.args
        self.logger.info(
            f"--Running test_case {tc.name} from {self.file_name}")
        tc.run(self.log_dir)
        self.logger.info(
            f"--Completed test_case {tc.name} from {self.file_name}")

    def run_test_module_cleanup(self):
        test_module_cleanup_tc = self._find_test_case("test_module_cleanup")
        if not test_module_cleanup_tc:
            return
        if self.test_module_setup_tc:
            # pass the output of the module_setup, so the cleanup can take care of any cleanup
            # required for the output
            test_module_cleanup_tc.test_module_setup_output = self.test_module_setup_tc.output
        self.logger.info(
            f"--Running test_module_cleanup from {self.file_name}")
        test_module_cleanup_tc.run(self.log_dir)
        self.logger.info(
            f"--Completed test_module_cleanup from {self.file_name}")

    def run_test_cases(self, log_dir) -> None:
        self.log_dir = log_dir
        os.chdir(os.path.dirname(self.file_name))
        self.start_time = datetime.datetime.now()
        self.run_test_module_setup()
        if self.test_module_setup_tc and self.test_module_setup_tc.status != "passed":
            # dont run other test cases if the module's setup failed
            self.logger.info(f"--Skipping test case file {self.file_name}")
            return
        for tc in self.get_test_cases():
            self.run_test_case(tc)
        self.run_test_module_cleanup()
        self.end_time = datetime.datetime.now()
        self.duration = (self.end_time - self.start_time).total_seconds()

    def to_json(self):
        # test case has its own json encoder
        return {
            'file_name': self.file_name,
            'args': self.args.__dict__ or {},
            'log_dir': self.log_dir,
            'start_time': str(self.start_time),
            'end_time': str(self.end_time),
            'duration': self.duration,
            'test_cases': self.get_test_cases()
        }


import models.shared as shared
import tensorflow as tf
import tensorflow.contrib.slim as slim
import models.base as base
from collections import OrderedDict

flags = tf.app.flags
FLAGS = flags.FLAGS

class HearModel(object):

    def __init__(self, input_shape=None, num_classes=14, embedding=1):

        self.scope = 'hear_net'
        self.num_classes = num_classes
        self.height = input_shape[0]
        self.width = input_shape[1]
        self.channels = input_shape[2]
        self.embedding = embedding

    def init_model(self, session, checkpoint_file):
        """
        Initializes DualCam-Net network parameters.
        """

        variables_to_restore = slim.get_model_variables(self.scope)

        init_fn = slim.assign_from_checkpoint_fn(checkpoint_file, variables_to_restore)
        init_fn(session)

    def _build_arg_scope(self, weight_decay=0.0005):
        with slim.arg_scope([slim.layers.conv2d, slim.layers.fully_connected],
                            activation_fn=slim.nn_ops.relu,
                            weights_regularizer=slim.regularizers.l2_regularizer(weight_decay),
                            biases_initializer=slim.init_ops.zeros_initializer()):
            with slim.arg_scope([slim.layers.conv2d], padding='SAME') as arg_sc:
                return arg_sc

    def _build_network(self, inputs, num_classes=14, is_training=True, keep_prob=0.5, scope='hear_net'):
        """
        Builds a three-layer network that operates over a spectrogram.
        """

        with tf.variable_scope(scope, 'hear_net', [inputs]) as sc:
            end_points_collection = sc.original_name_scope + '_end_points'

            # Collect outputs for convolution2d and max_pool2d
            with slim.arg_scope([slim.layers.conv2d, slim.layers.max_pool2d],
                                outputs_collections=[end_points_collection]):
                net = slim.layers.conv2d(inputs, 128, [11, 1], [1, 1], scope='conv1')
                net = slim.layers.max_pool2d(net, [5, 1], [5, 1], scope='pool1')
                net = slim.layers.conv2d(net, 256, [5, 1], [1, 1], scope='conv2')
                net = slim.layers.max_pool2d(net, [5, 1], [5, 1], scope='pool2')
                net = slim.layers.conv2d(net, 256, [3, 1], [1, 1], scope='conv3')
                net = slim.layers.max_pool2d(net, [5, 1], [5, 1], scope='pool3')
                net = slim.layers.conv2d(net, 1024, [1, 1], padding='VALID', scope='conv4')
                net = slim.layers.conv2d(net, 1024, 1, scope='conv5')

        # Convert end_points_collection into a end_point dictionary
        end_points = slim.layers.utils.convert_collection_to_dict(end_points_collection)

        return net, end_points

    def _build_model(self, spectrogram):
        """
        Builds the model using slim.
        """

        is_training = tf.placeholder(tf.bool, name='is_training')
        keep_prob = tf.placeholder(tf.float32, name='keep_prob')

        end_points = OrderedDict({
            'input': spectrogram,
            'is_training': is_training,
            'keep_prob': keep_prob
        })
        with slim.arg_scope(self._build_arg_scope(weight_decay=5e-4)):
            hearnet_output, hearnet_end_points = self._build_network(spectrogram,
                                                                     num_classes=self.num_classes,
                                                                     is_training=is_training,
                                                                     keep_prob=keep_prob,
                                                                     scope=self.scope)

        end_points.update(hearnet_end_points)

        shared_net_output, shared_net_end_points = shared.shared_net(hearnet_output,
                                                                     num_classes=self.num_classes,
                                                                     is_training=is_training,
                                                                     keep_prob=keep_prob,
                                                                     spatial_squeeze=True,
                                                                     scope=self.scope, embedding=self.embedding)

        end_points.update(shared_net_end_points)
        self.output = shared_net_output
        self.network = end_points
        self.train_vars = slim.get_trainable_variables(self.scope)

class DualCamHybridModel(object):

    def __init__(self, input_shape=None, num_classes=14, num_frames=12, embedding=1):

        self.scope = 'DualCamNet'

        self.num_classes = num_classes
        self.num_frames = num_frames

        self.height = input_shape[0]
        self.width = input_shape[1]
        self.channels = input_shape[2]
        self.embedding = embedding
        #self.output, self.network = self._build_model()

        #self.train_vars = slim.get_trainable_variables(self.scope)

    def init_model(self, session, checkpoint_file):
        """
        Initializes DualCam-Net network parameters.
        """

        # Restore all the model layers
        variables_to_restore = slim.get_variables(self.scope) #+ slim.get_variables('shared_net')

        # Initialization operation of the pre-trained weights
        init_fn = slim.assign_from_checkpoint_fn(checkpoint_file, variables_to_restore)

        # Load the pre-trained weights
        init_fn(session)

    def _build_network_slim(self, inputs, spatial_squeeze=False, scope='DualCamNet'):
        """
        Builds a DualCamNet network for classification using a 3D temporal convolutional layer with 7x1x1 filters.
        """

        with tf.variable_scope(scope, 'DualCamNet', [inputs]) as sc:
            end_points_collection = sc.original_name_scope + '_end_points'

            # Collect outputs for convolution2d and max_pool2d
            with slim.arg_scope([slim.layers.conv2d, slim.layers.max_pool2d], outputs_collections=[end_points_collection]):
                # ----------- 1st layer group ---------------
                net = tf.reshape(inputs, shape=(-1, self.num_frames, self.height, self.width, self.channels))
                net = slim.conv3d(net, self.channels, [7, 1, 1], scope='conv1', padding='SAME')
                net = tf.reshape(net, shape=(-1, self.height, self.width, self.channels))
                # ----------- 2nd layer group ---------------
                net = slim.conv2d(net, 32, [5, 5], scope='conv2', padding='SAME')
                net = slim.max_pool2d(net, [2, 2], scope='pool2')
                # ----------- 3rd layer group ---------------
                net = slim.conv2d(net, 64, [5, 5], scope='conv3', padding='SAME')
                net = slim.max_pool2d(net, [2, 2], scope='pool3')
                # ----------- 4th layer group ---------------
                # Use convolution2d instead of fully_connected layers
                net = slim.conv2d(net, 1024, 9, 12, scope='fc1', padding='VALID')

                # Convert end_points_collection into a end_point dictionary
                end_points = slim.layers.utils.convert_collection_to_dict(end_points_collection)

                if spatial_squeeze:
                    net = tf.squeeze(net, [1, 2], name='fc1/squeezed')
                    end_points[sc.name + '/fc1'] = net

                return net, end_points

    def _build_network(self, inputs, scope='DualCamNet'):
        """
        Builds a DualCamNet network for classification using a 3D temporal convolutional layer with 7x1x1 filters.
        """

        with tf.variable_scope(scope, 'DualCamNet', [inputs]):
            # ----------- 1st layer group ---------------
            inputs_reshaped = tf.reshape(inputs, shape=(-1, self.num_frames, self.height, self.width, self.channels))
            conv1 = base.build3DConvolution(inputs_reshaped, self.channels, self.channels, 1, 1, 7, name_scope='conv1',
                                            padding='SAME')
            relu1 = base.buildReLU(conv1, 'conv1')
            relu1_reshaped = tf.reshape(relu1, shape=(-1, self.height, self.width, self.channels))
            # ----------- 2nd layer group ---------------
            conv2 = base.build2DConvolution(relu1_reshaped, self.channels, 32, 5, 5, name_scope='conv2', padding='SAME')
            relu2 = base.buildReLU(conv2, 'conv2')
            pool2 = base.buildMaxPooling(relu2, 3, 3, 3, 3, 'conv2')  # 2 2 2 2
            # ----------- 3rd layer group ---------------
            conv3 = base.build2DConvolution(pool2, 32, 128, 5, 5, name_scope='conv3', padding='SAME')  # 64
            relu3 = base.buildReLU(conv3, 'conv3')
            if self.embedding == False:
                pool3 = tf.reduce_sum(relu3, axis=[1, 2])
                full1 = base.buildFullyConnected(pool3, 128, self.num_classes, name_scope='full1')
                # pool3 = base.buildMaxPooling(relu3, 2, 2, 2, 2, 'conv3')
                # ----------- 4th layer group ---------------
                # full1 = base.build2DConvolution(pool3, self.num_classes, 1024, 8, 6, name_scope='full1', padding='VALID')#12 9
                return full1, OrderedDict({  # full1
                    1: conv1,
                    2: relu1,
                    3: conv2,
                    5: relu2,
                    4: pool2,
                    6: conv3,
                    7: relu3,
                    8: pool3,
                    9: full1
                })
        return relu3, OrderedDict({  # full1
            1: conv1,
            2: relu1,
            3: conv2,
            5: relu2,
            4: pool2,
            6: conv3,
            7: relu3  # ,
            # 8: pool3,
            # 9: full1
        })

    def _build_model(self, acoustic_images):
        """
        Builds the hybrid model using slim and base functions.
        """

        # acoustic_images = tf.placeholder(tf.float32, [None, self.height, self.width, self.channels], name='acoustic_images')
        is_training = tf.placeholder(tf.bool, name='is_training')
        keep_prob = tf.placeholder(tf.float32, name='keep_prob')

        end_points = OrderedDict({
            'input': acoustic_images,
            'is_training': is_training,
            'keep_prob': keep_prob
        })

        dualcam_net_output, dualcam_net_end_points = self._build_network(acoustic_images,
                                                                         scope=self.scope)

        end_points.update(dualcam_net_end_points)

        # shared_net_output, shared_net_end_points = shared.shared_net(dualcam_net_output,
        # num_classes=self.num_classes,
        # is_training=is_training,
        # keep_prob=keep_prob,
        # spatial_squeeze=True,
        # scope=self.scope)

        # end_points.update(shared_net_end_points)

        self.output = dualcam_net_output  # shared_net_output
        self.network = end_points

        self.train_vars = slim.get_trainable_variables(self.scope)
        # return shared_net_output, end_points


import os
from os.path import basename
import shutil
from mutagen.mp3 import MP3

for subdir, dirs, files in os.walk(r'srcFile'):
    for file in files:
        #print os.path.join(subdir, file)
        filepath = subdir + os.sep + file

        if filepath.endswith(".mp3"):
            # Saving the length of the song property.
            audio = MP3(filepath)
            # Saving the directory name which we will use later to change
            # the name of the songs files(asthetically more pleasing.
            dirname = basename(os.path.dirname(filepath))
            dirname = dirname.split(' ', 1)[1]
            if audio.info.length > 29 :
                #Copying the file in the destination directory.
                shutil.copy(filepath,"D:\songs")
                destfile= os.path.join("D:\songs",file)
                dirname=os.path.join("D:\songs",dirname)
                # Renaming the files.
                os.rename(destfile,dirname + '.mp3')


# -*- coding: utf-8 -*-

from eightpuzzle import PuzzleState
from utils import goalPrint
import math

import queue as Q
import time

def bfs_search(initial_state,goal_state):

    """BFS search"""
    time1 = time.time()
    max_search_depth = 0
    expanded = -1
    frontier = Q.Queue()
    frontierDict = {}

    frontier.put(initial_state)
    while not frontier.empty() :
        #REMOVE
        state = frontier.get()
        frontierDict[state.config] = state
        state.expand()

        expanded += 1

        #GOAL CHECK
        if state.config == goal_state.config:

            goalPrint(state,expanded)
            print('max_search_depth: ',max_search_depth)
            print('running_time: ', time.time()-time1)

            return state

        state_neighbours = [state.move_up(),state.move_down(),state.move_left(),state.move_right()]
        state_neighbours =  [x for x in state_neighbours if x is not None]
#        state_neighbours.reverse()
        #EXPAND
        for i in state_neighbours:
            if i.config not in frontierDict:
                # these are the node in the queue, not the expanded nodes sucker
                frontier.put(i)
                frontierDict[i.config] = i

            if i.cost >= max_search_depth :
                    max_search_depth = i.cost

def main():

    begin_state = (0,8,7,6,5,4,3,2,1)
    #  1,2,5,3,4,0,6,7,8
    size = int(math.sqrt(len(begin_state)))

    goal_state = PuzzleState((0,1,2,3,4,5,6,7,8),size)
    hard_state = PuzzleState(begin_state, size)

    bfs_search(hard_state,goal_state)

if __name__ == '__main__':
    main()


# Blank Lambda function ref: https://github.com/awsdocs/aws-lambda-developer-guide/blob/main/sample-apps/blank-python/function/lambda_function.py
import logging
import os
import boto3  # type: ignore
import jsonpickle  # type: ignore

logger = logging.getLogger()
logger.setLevel(logging.INFO)

queue_url = os.environ["QUEUE_URL"]
sqs = boto3.client('sqs')
tasks = ["Hello from Lambda #1!", "Hello from Lambda #2!"]
logger.info('## TASKS\r' + jsonpickle.encode(tasks))

def handler(event, context):
    logger.info('## ENVIRONMENT VARIABLES\r' + jsonpickle.encode(dict(**os.environ)))
    logger.info('## EVENT\r' + jsonpickle.encode(event))
    logger.info('## CONTEXT\r' + jsonpickle.encode(context))

    # Send tasks to queue
    for task in tasks:
        try:
            response = sqs.send_message(
                QueueUrl=queue_url,
                MessageBody=(task),
                MessageGroupId='1337',
            )
            logger.info('## SQS RESPONSE\r' + jsonpickle.encode(response))
        except Exception as e:
            logger.error(e)
            raise e


# -*- coding: utf-8 -*-

import sys
import os
import errno
import time
import re

# global configuration
import vprimer.glv as glv
import vprimer.utils as utl

from vprimer.logging_config import LogConf
log = LogConf.open_log(__name__)

import pandas as pd
import vcfpy
import subprocess as sbp

from joblib import Parallel, delayed
#import dill as pickle

from vprimer.product import Product
from vprimer.allele_select import AlleleSelect
from vprimer.eval_variant import EvalVariant
from vprimer.variant import Variant
from vprimer.inout_primer3 import InoutPrimer3
from vprimer.blast import Blast

class Primer(object):

    def __init__(self):

        pass

    def construct_primer(self):

        proc_name = "primer"
        log.info("-------------------------------")
        log.info("Start processing {}\n".format(proc_name))

        # stop, action, gothrough
        ret_status = utl.decide_action_stop(proc_name)

        if ret_status == "stop":
            msg = "STOP. "
            msg += "Current process \'{}\' ".format(proc_name)
            msg += "has exceeded the User-specified stop point "
            msg += "\'{}', ".format(glv.conf.stop)
            msg += "so stop program. exit."
            log.info(msg)
            sys.exit(1)

        elif ret_status == "gothrough":
            msg = "SKIP \'{}\' proc, ".format(proc_name)
            msg += "glv.conf.progress = {}, ".format(glv.conf.progress)
            msg += "glv.conf.stop = {}, ".format(glv.conf.stop)
            msg += "so skip program."
            log.info(msg)
            return

        # for each distinguish_groups
        for proc_cnt, distin_dict in enumerate(glv.outlist.distin_files, 1):

            # logging current target
            utl.print_distin_info("primer", distin_dict, proc_cnt)

            marker_file = distin_dict['marker']['out_path']
            df_distin = pd.read_csv(
                marker_file, sep='\t', header=0, index_col=None)

            out_txt_file = distin_dict['primer']['out_path']
            utl.save_to_tmpfile(out_txt_file)

            with open(out_txt_file, mode='a') as f:
                # write header
                #f.write("{}\n".format(distin_dict['primer']['hdr_text']))

                start = time.time()

                if glv.conf.parallel == True:
                    log.info(
                        "do Parallel cpu {}, parallel {} blast {}".format(
                            glv.conf.thread,
                            glv.conf.parallel_blast_cnt,
                            glv.conf.blast_num_threads))

                    Parallel(
                        n_jobs=glv.conf.parallel_blast_cnt,
                        backend="threading")(
                        [
                            delayed(self._loop_primer3_check_blast) \
                                (distin_dict, marker_df_row, f) \
                                for marker_df_row in df_distin.itertuples()
                        ]
                    )

                else:
                    log.info("do Serial cpu {} / serial {} blast {}".format(
                        glv.conf.thread,
                        1,
                        glv.conf.blast_num_threads))

                    for marker_df_row in df_distin.itertuples():

                        self._loop_primer3_check_blast(
                            distin_dict, marker_df_row, f)

            utl.sort_file(
                'primer', distin_dict, out_txt_file,
                'chrom', 'pos', 'try_cnt', 'number')

            log.info("primer {} > {}.txt\n".format(
                utl.elapsed_time(time.time(), start),
                distin_dict['primer']['base_nam']))

    def _loop_primer3_check_blast(self, distin_dict, marker_df_row, f):

        prinfo = PrimerInfo()
        prinfo.prepare_from_marker_file(distin_dict, marker_df_row)

        prinfo.get_excluded_region()
        #print("stop")
        #sys.exit(1)

        prinfo.make_p3_info()
        blast_check_result_list = list()

        line = ''
        try_cnt = 0
        blast_check = ''

        while True:

            #log.debug("<try_cnt {}> SEQUENCE_EXCLUDED_REGION={}".format(
            #    try_cnt,
            #    iopr3.get_sequence_excluded_region()))
            try_cnt += 1

            # primer3
            self._do_primer3_pipe(prinfo.iopr3)

            #log.debug("")
            #log.debug("{}".format(prinfo.iopr3.get_primer3_out()))

            #log.debug("\nP3_COMMENT={}".format(iopr3.get_p3_comment()))

            # break if cann't generate primer
            if prinfo.iopr3.PRIMER_ERROR != '':
                #log.info("try_cnt {} {} PRIMER_ERROR={}".format(
                #    try_cnt,
                #    prinfo.iopr3.get_sequence_id(),
                #    prinfo.iopr3.PRIMER_ERROR))
                break

            elif prinfo.iopr3.PRIMER_PAIR_NUM_RETURNED == 0:
                #log.info("try_cnt {} {} PRIMER_PAIR_NUM_RETURNED={}".format(
                #    try_cnt,
                #    prinfo.iopr3.get_sequence_id(),
                #    prinfo.iopr3.PRIMER_PAIR_NUM_RETURNED))
                break

            # calc abs pos
            abs_left_stt, abs_left_end, \
            abs_right_stt, abs_right_end = \
                self._get_primer_pos_info(prinfo)

            # make fasta id
            left_fasta_id = self._make_primer_name(
                prinfo.chrom, abs_left_stt, abs_left_end, "plus")
            right_fasta_id = self._make_primer_name(
                prinfo.chrom, abs_right_stt, abs_right_end, "minus")

            prinfo.iopr3.set_primer_name(left_fasta_id, right_fasta_id)

            # search my blastn-short
            blast_check_result_list = \
                Blast.primer_blast_check(
                    left_fasta_id, right_fasta_id,
                    prinfo.iopr3.get_primer_left_seq(),
                    prinfo.iopr3.get_primer_right_seq())

            # break if complete
            if len(blast_check_result_list) == 0:

                #log.info("complete try_cnt {} {}".format(
                #    try_cnt, prinfo.iopr3.get_sequence_id()))

                #------------------------------
                complete = 1
                line, blast_check = self._primer_complete_to_line(
                    complete, blast_check_result_list, prinfo, try_cnt)
                f.write('{}\n'.format(line))
                break

            else:
                # go to next chance to add the primer pos to ex
                prinfo.iopr3.add_ex_region(prinfo.iopr3.get_primer_region())

                #log.info("next try_cnt {} {}".format(
                #    try_cnt,
                #    prinfo.iopr3.get_sequence_excluded_region()))

                #------------------------------
                complete = 0
                line, blast_check = self._primer_complete_to_line(
                    complete, blast_check_result_list, prinfo, try_cnt)
                f.write('{}\n'.format(line))

        if len(blast_check_result_list) != 0:
            log.info("skipped try_cnt {} {} {}".format(
                try_cnt,
                prinfo.iopr3.get_sequence_id(),
                blast_check))
           # log.info(iopr3.p3_out)

#        if line != '':
#            f.write('{}\n'.format(line))
#            # if you need
#            f.flush()

    def _primer_complete_to_line(
        self, complete, blast_check_result_list, prinfo, try_cnt):

        # blast_check
        blast_check = "-"
        add_cnt = len(blast_check_result_list) - 1
        if add_cnt == -1:
            pass
        elif add_cnt == 0:
            blast_check = "{}".format(blast_check_result_list[0])
        else:
            blast_check = "{}(+{})".format(
                blast_check_result_list[0], add_cnt)

        l_list = list()

        # to primer out file
        l_list += [prinfo.marker_id]
        # --------------
        l_list += [prinfo.chrom]
        l_list += [prinfo.pos]
        l_list += [prinfo.targ_grp]
        l_list += [prinfo.targ_ano]
        l_list += [prinfo.vseq_gno_str]
        l_list += [prinfo.gts_segr_lens]
        l_list += [prinfo.var_type]
        l_list += [prinfo.set_enz_cnt]
        # --------------
        l_list += [prinfo.marker_info]
        l_list += [prinfo.vseq_lens_ano_str]
        # --------------
        l_list += [prinfo.g0_seq_target_len]
        l_list += [prinfo.g0_seq_target]
        l_list += [prinfo.g1_seq_target_len]
        l_list += [prinfo.g1_seq_target]

        l_list += [prinfo.seq_template_ref_len]
        l_list += [prinfo.seq_template_ref_abs_pos]
        l_list += [prinfo.seq_template_ref_rel_pos]
        # --------------
        l_list += [try_cnt]
        l_list += [complete]
        l_list += [blast_check]
        # --------------

        l_list += [prinfo.iopr3.get_primer_product_size()]

        l_list += [prinfo.iopr3.get_primer_left()]
        l_list += [prinfo.iopr3.get_primer_left_id()]
        l_list += [prinfo.iopr3.get_primer_left_seq()]

        l_list += [prinfo.iopr3.get_primer_right()]
        l_list += [prinfo.iopr3.get_primer_right_id()]
        l_list += [prinfo.iopr3.get_primer_right_seq()]

        l_list += [prinfo.SEQUENCE_TARGET]
        #
        l_list += [prinfo.iopr3.get_sequence_excluded_region()]
        l_list += [prinfo.seq_template_ref]

        return '\t'.join(map(str, l_list)), blast_check

    def _make_primer_name(
            self, chrom, abs_primer_stt_pos, abs_primer_end_pos, strand):

        # {NC_028450.1}44676.44700.plus
        #primer_name = "{{{}}}{}.{}.{}".format(
        #    chrom,
        #    abs_primer_stt_pos,
        #    abs_primer_end_pos,
        #    strand)

        # NC_028450.1:44676-44700:plus
        primer_name = "{}:{}-{}:{}".format(
            chrom,
            abs_primer_stt_pos,
            abs_primer_end_pos,
            strand)

        return primer_name

    def _do_primer3_pipe(self, iopr3):

        # exec primer3 through pipe
        primer3_in = iopr3.get_p3_input()

        primer3_out_p = sbp.Popen(
            ['primer3_core'],
            stdin=sbp.PIPE,
            stdout=sbp.PIPE)

        primer3_out = primer3_out_p.communicate(
            primer3_in.encode())[0].decode()

        iopr3.set_primer3_out(primer3_out)

    def _get_primer_pos_info(self, prinfo):
        '''
        '''

        # This value brings PRIMER_LEFT_0 value directly.
        # 353,25
        PRIMER_LEFT_0_stt, PRIMER_LEFT_0_len = \
            prinfo.iopr3.get_primer_left_info()

        # For PRIMER_LEFT_0, the starting point is reported
        # in the 5'-3'direction as usual.
        # Therefore, the absolute position is calculated
        # based on the length as usual.
        abs_PRIMER_LEFT_0_stt = self._get_abspos(
            PRIMER_LEFT_0_stt, prinfo.abs_frag_pad_pre_stt)
        abs_PRIMER_LEFT_0_end = abs_PRIMER_LEFT_0_stt + PRIMER_LEFT_0_len - 1

        # This value brings PRIMER_RIGHT_0 value directly.
        # 567,25
        # On the other hand, for PRIMER_RIGHT_0,
        # the position on the 3'side is reported.
        PRIMER_RIGHT_0_end, PRIMER_RIGHT_0_len = \
            prinfo.iopr3.get_primer_right_info()

        # Therefore, the starting point is obtained using the length.
        PRIMER_RIGHT_0_stt = PRIMER_RIGHT_0_end - PRIMER_RIGHT_0_len + 1
        # After that, just convert to absolute position.
        abs_PRIMER_RIGHT_0_stt = self._get_abspos(
            PRIMER_RIGHT_0_stt, prinfo.abs_frag_pad_pre_stt)
        abs_PRIMER_RIGHT_0_end = \
            abs_PRIMER_RIGHT_0_stt + PRIMER_RIGHT_0_len - 1

        return \
            abs_PRIMER_LEFT_0_stt, abs_PRIMER_LEFT_0_end, \
            abs_PRIMER_RIGHT_0_stt, abs_PRIMER_RIGHT_0_end

    def _get_abspos(self, ref_pos, abs_template_stt):

        # 11
        #  1234567890
        #        7 11+7=18 -1

        return abs_template_stt + ref_pos - 1

class PrimerInfo(object):

    def __init__(self):

        self.marker_id = ''

        self.chrom = ''
        self.pos = 0

        self.targ_grp = ''
        self.g0_name = ''
        self.g1_name = ''

        self.gts_segr_lens = ''

        self.targ_ano = ''
        self.g0_ano = -1
        self.g1_ano = -1

        self.vseq_gno_str = ""

        self.set_enz_cnt = ''
        self.var_type = ''
        self.marker_info = ''

        self.vseq_lens_ano_str = ''

        self.target_gno = -1
        self.target_len = 0
        self.enzyme_name = ''
        self.digest_pattern = ''

        self.g0_seq_target_len = 0
        self.g0_seq_target = ''
        self.g1_seq_target_len = 0
        self.g1_seq_target = ''
        self.seq_template_ref_len = 0
        self.seq_template_ref_abs_pos = ''
        self.seq_template_ref_rel_pos = ''
        self.SEQUENCE_TARGET = ''
        self.seq_template_ref = ''

        # abs template info
        self.abs_frag_pad_pre_stt = 0
        self.abs_frag_pad_pre_end = 0
        self.abs_around_seq_pre_stt = 0
        self.abs_around_seq_pre_end = 0
        self.abs_pos = 0
        self.abs_around_seq_aft_stt = 0
        self.abs_around_seq_aft_end = 0
        self.abs_frag_pad_aft_stt = 0
        self.abs_frag_pad_aft_end = 0

        # abs template info
        self.rel_frag_pad_pre_stt = 0
        self.rel_frag_pad_pre_end = 0
        self.rel_around_seq_pre_stt = 0
        self.rel_around_seq_pre_end = 0
        self.rel_pos = 0
        self.rel_around_seq_aft_stt = 0
        self.rel_around_seq_aft_end = 0
        self.rel_frag_pad_aft_stt = 0
        self.rel_frag_pad_aft_end = 0

        self.SEQUENCE_EXCLUDED_REGION = ''

        self.p3_comment = ''

    def prepare_from_marker_file(self, distin_dict, marker_df_row):

        hdr_dict = distin_dict['marker']['hdr_dict']

        # basic
        self.marker_id, \
        self.chrom, \
        self.pos, \
        self.targ_grp, \
        self.g0_name, \
        self.g1_name, \
        self.targ_ano, \
        self.g0_ano, \
        self.g1_ano, \
        self.vseq_gno_str, \
        self.gts_segr_lens, \
        self.var_type, \
        self.set_enz_cnt, \
        self.marker_info, \
        self.vseq_lens_ano_str, \
        self.enzyme_name, \
        self.digest_pattern, \
        self.target_gno, \
        self.target_len = \
            utl.get_basic_primer_info(marker_df_row, hdr_dict)

        self.g0_seq_target_len = \
            int(marker_df_row[hdr_dict['g0_seq_target_len']])
        self.g0_seq_target = \
            str(marker_df_row[hdr_dict['g0_seq_target']])
        self.g1_seq_target_len = \
            int(marker_df_row[hdr_dict['g1_seq_target_len']])
        self.g1_seq_target = \
            str(marker_df_row[hdr_dict['g1_seq_target']])

        self.seq_template_ref_len = \
            int(marker_df_row[hdr_dict['seq_template_ref_len']])
        self.seq_template_ref_abs_pos = \
            str(marker_df_row[hdr_dict['seq_template_ref_abs_pos']])

        #log.debug("{}".format(self.seq_template_ref_abs_pos))

        self.seq_template_ref_rel_pos = \
            str(marker_df_row[hdr_dict['seq_template_ref_rel_pos']])

        #log.debug("{}".format(self.seq_template_ref_rel_pos))

        self.SEQUENCE_TARGET = \
            str(marker_df_row[hdr_dict['SEQUENCE_TARGET']])
        self.seq_template_ref = \
            str(marker_df_row[hdr_dict['seq_template_ref']])

        # abs template info
        self.abs_frag_pad_pre_stt, \
        self.abs_frag_pad_pre_end, \
        self.abs_around_seq_pre_stt, \
        self.abs_around_seq_pre_end, \
        self.abs_pos, \
        self.abs_around_seq_aft_stt, \
        self.abs_around_seq_aft_end, \
        self.abs_frag_pad_aft_stt, \
        self.abs_frag_pad_aft_end = \
            Product.separate_seq_template_pos(
                self.seq_template_ref_abs_pos)

        # rel template info
        self.rel_frag_pad_pre_stt, \
        self.rel_frag_pad_pre_end, \
        self.rel_around_seq_pre_stt, \
        self.rel_around_seq_pre_end, \
        self.rel_pos, \
        self.rel_around_seq_aft_stt, \
        self.rel_around_seq_aft_end, \
        self.rel_frag_pad_aft_stt, \
        self.rel_frag_pad_aft_end = \
            Product.separate_seq_template_pos(
                self.seq_template_ref_rel_pos)

    def _get_relpos(self, abs_pos):

        #   | self.abs_frag_pad_pre_stt
        #                 self.abs_frag_pad_aft_end|
        #   <-------><========>P<=========><------->
        #            |self.abs_around_seq_pre_stt
        #      self.abs_around_seq_aft_end|
        #   |101 109|
        #   123456789
        #       |105
        #   105-101+1 = 5

        rel_pos = abs_pos - self.abs_frag_pad_pre_stt + 1

        return rel_pos

    def get_excluded_region(self):

        SEQUENCE_EXCLUDED_REGION = list()

        #logf_l = ["{} self.pos={} rel_pos={} rel_end_pos={} "]
        #logf_l += ["region_len={} template_len={}"]
        #logf = "".join(logf_l)

        region = "{}:{}-{}".format(
            self.chrom,
            self.abs_frag_pad_pre_stt,
            self.abs_frag_pad_aft_end)

        reader = vcfpy.Reader.from_path(glv.conf.vcf_file_path)
        vcf_ittr = reader.fetch(region)

        # access to vcf using iterater
        for record in vcf_ittr:
            sample0 = glv.conf.group_members_dict[self.g0_name][0]
            sample1 = glv.conf.group_members_dict[self.g1_name][0]

            # もし、サンプル間でvariantが見つかった場合は、
            s0_0, s0_1, s1_0, s1_1 = \
                AlleleSelect.record_call_for_sample(record, sample0, sample1)

            if utl.is_same_gt(s0_0, s0_1, s1_0, s1_1) == False:

                # 20200713 here
                if self.pos != record.POS:
                    rel_pos = self._get_relpos(record.POS) #
                    # そのポジションのrefのvseq分を登録する
                    # 長さがfragment長を超える場合は、
                    # 調整する。
                    # 見つかったのはPOS
                    # REFのlength

                    # self.pos|
                    #     1036|
                    # ATGCATGCA ref_len=1
                    #         T
                    #         C
                    #         1036 + 1 - 1
                    region_len = len(record.REF)
                    rel_end_pos = rel_pos + region_len - 1

                    #  pos   len     end
                    #  1036 (10)     1045
                    #            1041 temp_len

#                    log.debug(logf.format(
#                        1, self.pos, rel_pos, rel_end_pos, region_len,
#                        self.seq_template_ref_len))

                    #log.debug("{}, {}".format(
                    #    rel_end_pos, self.seq_template_ref_len))
                    #log.debug("{}, {}".format(
                    #    type(rel_end_pos), type(self.seq_template_ref_len)))

                    if rel_end_pos > self.seq_template_ref_len:
                        diff_len = rel_end_pos - self.seq_template_ref_len
                        region_len = region_len - diff_len

#                        log.debug(logf.format(
#                            2, self.pos, rel_pos, rel_end_pos, region_len,
#                            self.seq_template_ref_len))

                    SEQUENCE_EXCLUDED_REGION += [
                        "{},{}".format(rel_pos, region_len)]

        self.SEQUENCE_EXCLUDED_REGION = " ".join(SEQUENCE_EXCLUDED_REGION)
        #log.debug("SEQUENCE_EXCLUDED_REGION={}".format(
        #    self.SEQUENCE_EXCLUDED_REGION))

    def make_p3_info(self):

        # save information
        self.p3_comment = self._make_p3_comment()
        #log.debug("{}".format(self.p3_comment))

        self.iopr3 = InoutPrimer3()
        self.iopr3.set_p3_comment(self.p3_comment)
        self.iopr3.set_sequence_target(self.SEQUENCE_TARGET)
        self.iopr3.add_ex_region(self.SEQUENCE_EXCLUDED_REGION)
        self.iopr3.set_sequence_id(self.marker_id)
        self.iopr3.set_sequence_template(self.seq_template_ref)

        #log.debug("")
        #log.debug("{}".format(self.iopr3.get_p3_input()))

    def _make_p3_comment(self):

        p3_comment = list()
        p3_comment += ["{}:{}".format('marker_id', self.marker_id)]
        p3_comment += ["{}:{}".format('var_type', self.var_type)]
        p3_comment += ["{}:{}".format('g0_name', self.g0_name)]
        p3_comment += ["{}:{}".format('g1_name', self.g1_name)]
        p3_comment += ["{}:{}".format('marker_info', self.marker_info)]

        p3_comment += ["{}:{}".format(
            'g0_seq_target_len', self.g0_seq_target_len)]
        p3_comment += ["{}:{}".format(
            'g0_seq_target', self.g0_seq_target)]
        p3_comment += ["{}:{}".format(
            'g1_seq_target_len', self.g1_seq_target_len)]
        p3_comment += ["{}:{}".format(
            'g1_seq_target', self.g1_seq_target)]

        p3_comment += ["{}:{}".format(
            'seq_template_ref_len', self.seq_template_ref_len)]
        p3_comment += ["{}:{}".format(
            'seq_template_ref_abs_pos', self.seq_template_ref_abs_pos)]
        p3_comment += ["{}:{}".format(
            'seq_template_ref_rel_pos', self.seq_template_ref_rel_pos)]

        return ';'.join(map(str, p3_comment))


from collections import OrderedDict
import json

def dom_tree_printer(dom_tree):
    """
    - dom_tree: dict(str, node)
    """
    res = OrderedDict()
    keys_sorted = list(dom_tree.keys())
    keys_sorted.sort()
    for vertex in keys_sorted:
        l = list(dom_tree[vertex].succs)
        l.sort()
        res[vertex] = l
    print(json.dumps(res, indent=2))

def frontier_printer(frontier):
    """
    - frontier: dict(str, set(str))
    """
    res = OrderedDict()
    keys_sorted = list(frontier.keys())
    keys_sorted.sort()
    for vertex in keys_sorted:
        l = list(frontier[vertex])
        l.sort()
        res[vertex] = l
    print(json.dumps(res, indent=2))


import argparse
import json
import sys
import subprocess
import os
import gzip
import logging

def file_open(filename, mode="r"):
    if "gz" in filename:
        return gzip.open(filename, mode + "t")
    return open(filename, mode)

def shell_call(call):
    try:
       print(call)
       output = subprocess.check_output(call, universal_newlines=True, shell=True)
    except Exception as e:
        output = str(e.output)
    return output

def assert_file(filename, error_msg):
    if not os.path.isfile(filename):
        logging.error(error_msg)
        raise Exception()

def parse_video(video_file):
    ffprobe = os.path.normpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "FFmpeg_x64", "ffprobe"))
    assert_file(ffprobe, "local patched ffprobe could not be found, please run from correct location.")
    cmd = f""" {ffprobe} -v error -show_format -select_streams v:0 -show_frames -show_entries stream=bit_rate,width,height,avg_frame_rate,codec_name -show_entries frame=key_frame,pkt_size,pict_type -of json "{video_file}" """
    res = shell_call(cmd).split("\n")
    current_values = []
    grouped_qp_values = []
    remaining = []
    print(res)
    for l in res:
        if "start_frame" in l:  # frame start indication
            if current_values != []:
                grouped_qp_values.append(current_values)
                current_values = []
            continue
        if "qp=" in l:
            # its a qp=.. line
            current_values.append(int(l.split("=")[1].strip()))
        else:
            remaining.append(l)

    grouped_qp_values.append(current_values)

    def clean_str(r):
        return r.split(":")[1].split(",")[0].replace("\"", "").strip()

    # frame stats
    key_frames = []
    pkt_sizes = []
    pict_types = []
    bit_rate = "unknown"
    width = "unknown"
    height = "unknown"
    avg_frame_rate = "unknown"
    codec_name = "unknown"

    for r in remaining:
        if "key_frame" in r:
            key_frames.append(int(clean_str(r)))
        if "pkt_size" in r:
            pkt_sizes.append(int(clean_str(r)))
        if "pict_type" in r:
            pict_types.append(clean_str(r))
        if "bit_rate" in r:
            bit_rate = int(clean_str(r))
        if "width" in r:
            width = int(clean_str(r))
        if "height" in r:
            height = int(clean_str(r))
        if "avg_frame_rate" in r:
            avg_frame_rate = clean_str(r)
        if "codec_name" in r:
            codec_name = clean_str(r)

    per_frame = []
    for i, g in enumerate(grouped_qp_values):
        per_frame.append({
            "mean_qp": sum(g) / len(g),
            "min_qp": min(g),
            "max_qp": max(g),
            "pkt_size": pkt_sizes[i],
            "key_frame": key_frames[i],
            "pict_type": pict_types[i]
        })
    return {
        "per_frame": per_frame,
        "bit_rate": bit_rate,
        "width": width,
        "height": height,
        "avg_frame_rate": avg_frame_rate,
        "codec_name": codec_name
    }

def main(_):
    # argument parsing
    parser = argparse.ArgumentParser(description='extract qp values from hevc encoded videos',
                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument("video_file", type=str, help="input video file")
    parser.add_argument("--report_file", type=str, default="report.json.gz", help="file to store the report, supports compression, e.g. using report.gz")
    parser.add_argument("--nostdout", action="store_true", help="no report output on stdout")

    a = vars(parser.parse_args())

    report = parse_video(a["video_file"])
    if a["report_file"] is None or not a["nostdout"]:
        print(json.dumps(report, indent=4))
    if a["report_file"] is not None:
        with file_open(a["report_file"], "w") as rfp:
            json.dump(report, rfp, indent=4)

if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))


from common.storage.backend import StorageBackend
from shutil import copyfile
import os

class LocalFilesystem(StorageBackend):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.LOGGGER.info(f"Init local filesystem with {list(*args)} {list(**kwargs)}")
        if self.remote_prefix == "":
            if os.environ.get("STORAGE_DIR"):
                self.remote_prefix = os.environ.get("STORAGE_DIR")
            else:
                self.remote_prefix = "/tmp"

        if self.local_prefix == "":
            if os.environ.get("TMP_DIR"):
                self.local_prefix = os.environ.get("TMP_DIR")
            else:
                self.local_prefix = "/tmp"

    def put_object(self, src, dst=""):
        if dst == "":
            dst = os.path.join(self.remote_prefix, self._generate_tempname())
        copyfile(src, dst)
        return src, dst

    def get_object(self, src, dst=""):
        if dst == "":
            dst = os.path.join(self.local_prefix, self._generate_tempname())
        copyfile(src, dst)
        return src, dst