import datasets
from raft_baselines.classifiers import CohereClassifier, TransformersCausalLMClassifier
from sklearn.metrics import accuracy_score, f1_score

# prompt="""Label the sentence based on whether it is related to an adverse drug effect (ADE). Details are described below:
# Drugs: Names of drugs and chemicals that include brand names, trivial names, abbreviations and systematic names were annotated. Mentions of drugs or chemicals should strictly be in a therapeutic context. This category does not include the names of metabolites, reaction byproducts, or hospital chemicals (e.g. surgical equipment disinfectants).
# Adverse effect: Mentions of adverse effects include signs, symptoms, diseases, disorders, acquired abnormalities, deficiencies, organ damage or death that strictly occur as a consequence of drug intake.
# Possible labels:
# 1. ADE-related
# 2. not ADE-related

# Sentence: The mechanism by which sunitinib induces gynaecomastia is thought to be associated with an unknown direct action on breast hormonal receptors.
# Label:"""
# co.choose_best(model='baseline-shrimp',query=prompt, options=['ADE-related', 'not ADE-related'], mode='APPEND_OPTION')

class Pcolour:
    MAGENTA = '\033[95m'
    BLUE = '\033[94m'
    GREEN = '\033[92m'
    YELLOW = '\033[93m'
    RED = '\033[91m'
    END = '\033[0m'
    BOLD = '\033[1m'
    UNDERLINE = '\033[4m'
def colorize(text, color):
    return f"{color}{text}{Pcolour.END}"
def blue(*args):
    return colorize(" ".join(map(str, list(args))), Pcolour.BLUE)
def magenta(*args):
    return colorize(" ".join(map(str, list(args))), Pcolour.MAGENTA)
def red(*args):
    return colorize(" ".join(map(str, list(args))), Pcolour.RED)

config_kwargs = dict(
    # random_seed = 42,
    random_seed = 9385,
    # random_seed = 4387,
    zero_shot = False,
    should_print_prompt = True,

    # model = 'baseline-seal'
    # model = 'baseline-shark'
    # model = 'baseline-orca'
    # model_type = 'distilgpt2'
    # model_type = 'gpt2-xl'
    # model_type = 'EleutherAI/gpt-neo-2.7B'
    model = 'baseline-shrimp'
    # model = 'baseline-shrimp'
)

# scoring method, do semantic selection,

classifier_kwargs = dict(
    # n_test = 5,
    # scoring_method = 'first_token_likelihood' if not zero_shot else 'avg_log_likelihood',  # first_token_likelihood or avg_log_likelihood
    scoring_method = 'first_token_likelihood',  # first_token_likelihood or avg_log_likelihood
    # scoring_method = 'avg_log_likelihood',  # first_token_likelihood or avg_log_likelihood
    engine = config_kwargs['model'],
    # model_type = config_kwargs['model_type'],
    do_semantic_selection = False,
    max_tokens = 1024 if config_kwargs['model'] in ['baseline-shrimp', 'baseline-otter', 'baseline-seal'] else 2048
    # max_tokens = 1024 if config_kwargs['model_type'] in ['baseline-shrimp', 'baseline-otter', 'baseline-seal'] else 2048
    )

# if zero_shot:
#     print('USING AVG LIKELIHOOD BC ZERO SHOT')

configs = datasets.get_dataset_config_names("ought/raft")
configs.remove('banking_77')

# configs = ['twitter_complaints']

# configs = ['ade_corpus_v2']
# configs = ['banking_77']

NUM_EXAMPLES = {
    "ade_corpus_v2": 25,
    # "ade_corpus_v2": 4,
    "banking_77": 10,
    "terms_of_service": 5,
    "tai_safety_research": 5,
    "neurips_impact_statement_risks": 25,
    "overruling": 25,
    "systematic_review_inclusion": 10,
    "one_stop_english": 3, # TODO: normally 5 but was giving too long errors
    "tweet_eval_hate": 50,
    "twitter_complaints": 25,
    "semiconductor_org_types": 5,
}

# # TODO: uncomment for Cohere
if classifier_kwargs['max_tokens'] == 1024:
    NUM_EXAMPLES = {k: v//2 for k, v in NUM_EXAMPLES.items()}

# train = datasets.load_dataset(
#     "ought/raft", "neurips_impact_statement_risks", split="train"
# )

# import ipdb; ipdb.set_trace()

# acc_scores={}
# f1_scores={}
results = {}
results['classifier_kwargs'] = classifier_kwargs
results.update({'num_examples': NUM_EXAMPLES})

for config in configs:
    results[config] = {}
    print('\n', magenta(config),'\n')
    train_dataset = datasets.load_dataset("ought/raft", config, split="train")
    print(magenta(str(train_dataset.features['Label'])))
    for clas in train_dataset.features['Label'].names:
        print(clas, train_dataset.features['Label'].str2int(clas))
    extra_kwargs = {
        "config": config,
        "num_prompt_training_examples": NUM_EXAMPLES[config] if not config_kwargs['zero_shot'] else 0,
    }
    if config == "banking_77":
        extra_kwargs["add_prefixes"] = True

    # initialize with a dict
    results[config].update(extra_kwargs)

    # classifier = TransformersCausalLMClassifier(
    #     train_dataset, #config=train_dataset,
    #     **classifier_kwargs, **extra_kwargs
    # )
    # TODO change
    classifier = CohereClassifier(
        train_dataset, #config=train_dataset,
        **classifier_kwargs, **extra_kwargs
    )

    correct = 0
    n = 0
    probs = []
    y_pred = []
    y_true = []
    results[config]['num_failed'] = 0

    def compute_and_store_metrics(y_true, y_pred):
        global results
        acc = accuracy_score(y_true=y_true, y_pred=y_pred)
        f1 = f1_score(y_true=y_true, y_pred=y_pred, labels=range(1, train_dataset.features['Label'].num_classes), average='macro')
        micro_f1 = f1_score(y_true=y_true, y_pred=y_pred, labels=range(1, train_dataset.features['Label'].num_classes), average='micro')
        results[config]['acc'] = acc
        results[config]['f1'] = f1
        results[config]['micro_f1'] = micro_f1
        print(red('Accuracy:', acc))
        print(red('F1 (macro):', f1))
        print(red('Micro F1:', micro_f1))

    def predict(example):
        global correct, n, probs, y_pred, y_true
        # import ipdb; ipdb.set_trace()
        label_true = example["Label"]
        output_probs = classifier.classify(example, random_seed=config_kwargs['random_seed'], should_print_prompt=config_kwargs['should_print_prompt'])
        output = max(output_probs.items(), key=lambda kv_pair: kv_pair[1])
        probs.append(output_probs)

        label_pred = train_dataset.features["Label"].str2int(output[0])
        example["Label Prediction"] = label_pred
        print(magenta('Prediction:'), output[0])
        print(magenta('True:'), train_dataset.features["Label"].int2str(label_true))

        y_pred.append(label_pred)
        y_true.append(label_true)
        compute_and_store_metrics(y_true=y_true, y_pred=y_pred)

        # import ipdb; ipdb.set_trace()
        return example

    for i, example in enumerate(train_dataset):
        try:
            predict(example)
        except Exception as e:
            print(e)
            results[config]['num_failed'] += 1

        # if i ==3:
        #     break

    # train_dataset.map(predict, load_from_cache_file=False)

    # overwrite for final time
    compute_and_store_metrics(y_true=y_true, y_pred=y_pred)
    results[config].update({'y_true': y_true, 'y_pred':y_pred})

print('Accuracies')
print({config: results[config]['acc'] for config in configs})
print('F1 scores')
print({config: results[config]['f1'] for config in configs})
print('micro F1 scores')
print({config: results[config]['micro_f1'] for config in configs})

import json

from datetime import datetime
date = datetime.strftime(datetime.now(), '%Y%m%d_%H%M')
with open(f'cohere_results/{date}.json', 'w') as f:
    json.dump(results, f)


import os, sys
import numpy as np
from pdb import set_trace as st
import pandas as pd

dir =  f"models/prune/true_false_dist/allconv_weight"

def filter_csv_result(df):
    df = df[df['ratio'] == 0.5]
    posnegweight = df['posnegweight_small']
    return posnegweight

path = os.path.join(dir, f"allconv_class_all.csv")
df = pd.read_csv(path, index_col=0)
# print(df)

# auc all
# path = os.path.join(dir, f"allconv_class_auc_all.csv")
# df = pd.read_csv(path, index_col=0)
# df = df[df['ratio'] == 0.5]
# print(df)
# st()

# Compare between class trace and "all" trace on each class
path = os.path.join(dir, f"allconv_class_auc_all.csv")
all_df = pd.read_csv(path, index_col=0)
for class_id in range(10):
    path = os.path.join(dir, f"allconv_class_{class_id}.csv")
    class_df = pd.read_csv(path, index_col=0)
    class_df = class_df[class_df["test_class"] == class_id]
    class_df = class_df[["posnegweight_small", "ratio"]]
    class_df = class_df.rename(columns={"posnegweight_small": "class"})
    all_df_class = all_df[all_df["test_class"] == class_id]
    all_df_class = all_df_class[["posnegweight_small", "ratio"]]
    all_df_class = all_df_class.rename(columns={"posnegweight_small": "all"})
    df = pd.merge(class_df, all_df_class)
    print(class_id)
    print(df)
    print()
st()

# Auc between different classes
interclass_matrix = []
for class_id in list(range(10))+["auc_all"]:
    path = os.path.join(dir, f"allconv_class_{class_id}.csv")
    df = pd.read_csv(path, index_col=0)
    df['trace_class'] = class_id
    posnegweight = filter_csv_result(df).to_numpy()
    interclass_matrix.append(posnegweight)

interclass_matrix = np.stack(interclass_matrix)
print(interclass_matrix)
st()


# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved

import os
import unittest
import collections.abc

from hmalib.hashexchanges.api.non_threatexchange_api import NonThreatExchangeAPI
from hmalib.hashexchanges.api.non_threatexchange_api_representations import HashRecord

X_FUNCTIONS_KEY = os.getenv("X_FUNCTIONS_KEY")

OCP_APIM_SUBSCRIPTION_KEY = os.getenv("OCP_APIM_SUBSCRIPTION_KEY")

BASE_NON_THREATEXCHANGE_URL = os.getenv("BASE_NON_THREATEXCHANGE_URL")

@unittest.skipUnless(
    X_FUNCTIONS_KEY and OCP_APIM_SUBSCRIPTION_KEY and BASE_NON_THREATEXCHANGE_URL,
    "Integration Test requires two api keys and a base url.",
)
class NonThreatExchangeAPIIntegrationTest(unittest.TestCase):
    def setUp(self):
        self.api = NonThreatExchangeAPI(
            X_FUNCTIONS_KEY, OCP_APIM_SUBSCRIPTION_KEY, BASE_NON_THREATEXCHANGE_URL
        )

    def test_get_hashes(self):
        """
        Assumes that the response has at least one hash record.
        """
        response = self.api.get_hash_records(start_timestamp=0, page_size=10)
        #   test get_hash_records api from the very beginning to make sure we will get at least one record in the response.
        #   otherwise the test will fail.
        self.assertTrue(
            isinstance(response.hashRecords, collections.abc.Sequence)
            and not isinstance(response, staticmethod),
            "hashRecords should be a list!",
        )

        self.assertTrue(isinstance(response.hashRecords[0], HashRecord))


# uncompyle6 version 3.7.4
# Python bytecode 3.5 (3351)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: /home/kocal/Dev/Python/django-tornado-websockets/tornado_websockets/__init__.py
# Compiled at: 2016-12-19 16:32:09
# Size of source mod 2**32: 338 bytes
__version_info__ = ('0', '2', '2')
__version__ = '.'.join(__version_info__)

def django_app():
    import django.core.handlers.wsgi, tornado.wsgi
    django.setup()
    app = tornado.wsgi.WSGIContainer(django.core.handlers.wsgi.WSGIHandler())
    app = ('.*', tornado.web.FallbackHandler, dict(fallback=app))
    return app


"""This program handles the elev_025-deg.nc dataset

Source: http://research.jisao.washington.edu/data_sets/elevation/
TASK: Find original source and cite if necessary.
"""

import xarray as xr
import pandas as pd
import numpy as np

elev = xr.open_dataset("elev_025-deg.nc")

# Example of getting elevation for selected lat and lon
elev.sel(lat=-36.875, lon=144.375).data.values

# Read vic_posts df
def read_vic_postcodes():
    vic_posts = pd.read_csv("vic_postcodes.csv", "r",
    delimiter=",",
    index_col = 0)

    return vic_posts

vic_posts = read_vic_postcodes()

# Add elevation to vic_posts
def add_elevation_to_vicposts():
    altitudes = []
    for index in range(len(vic_posts)):
        lon, lat = vic_posts.iloc[index,:][2:4]

        # Round lat and lon to suit keys in elev dataset.
        lon = round((lon - 0.125) * 4) / 4 + 0.125
        lat = round((lat - 0.125) * 4) / 4 + 0.125

        altitudes.append(elev.sel(lat=lat, lon=lon).data.values[0])

    # Process altitude data
    altitudes = np.array(altitudes)
    altitudes[altitudes<0] = 0

    # Insert into vic_posts dataframe.
    vic_posts["Altitude"] = altitudes

add_elevation_to_vicposts()

# Test to verify it worked.
vic_posts

# Save as new csv file
vic_posts.to_csv("vic_postcodes_elev.csv")


# This file is part of the litprog project
# https://gitlab.com/mbarkhau/litprog
#
# Copyright (c) 2019 Manuel Barkhau (mbarkhau@gmail.com) - MIT License
# SPDX-License-Identifier: MIT
import io
import os
import re
import sys
import enum
import math
import time
import typing as typ
import logging
import os.path
import datetime as dt
import operator as op
import functools as ft
import itertools as it
import collections

import pathlib2 as pl

import litprog.session
from litprog.parse import Block
from litprog.parse import Context
from litprog.parse import Headline
from litprog.parse import Directive
from litprog.parse import MarkdownFile
from litprog.parse import MarkdownElement

log = logging.getLogger(__name__)

COLOR_CODE_RE = re.compile(r"\d+(;\d)?")

TERM_COLORS = {
    'black'  : "30",
    'red'    : "31",
    'green'  : "32",
    'yellow' : "33",
    'blue'   : "34",
    'magenta': "35",
    'cyan'   : "36",
    'white'  : "37",
}

CapturedLine = litprog.session.CapturedLine

class Capture(typ.NamedTuple):
    command    : str
    exit_status: int
    runtime    : float
    lines      : typ.List[CapturedLine]

def get_directive(block: Block, name: str) -> typ.Optional[Directive]:
    for directive in block.directives:
        if directive.name == name:
            return directive
    return None

def iter_directives(block: Block, name: str) -> typ.Iterable[Directive]:
    for directive in block.directives:
        if directive.name == name:
            yield directive

def has_directive(block: Block, name: str) -> bool:
    return get_directive(block, name) is not None

CONSTANT_RE = re.compile(r"`lp_const:\s*(?P<name>\w+)\s*=(?P<value>.*)`")

class ConstItem(typ.NamedTuple):
    decl_path: pl.Path
    name     : str
    value    : str

def _parse_constants(build_ctx: Context) -> typ.List[ConstItem]:
    constants: typ.List[ConstItem] = []
    for md_file in build_ctx.files:
        decl_path = md_file.md_path
        for elem in md_file.elements:
            for match in CONSTANT_RE.finditer(elem.content):
                name  = match.group('name')
                value = match.group('value')
                constants.append(ConstItem(decl_path, name, value))

    # Expand longer constants first, so constants that
    # are substrings of others don't clobber the longer
    # constant. This is not an ideal solution, but since
    # we are restricted to textual replacement with any
    # conceivable language this may be adequate.
    constants.sort(key=lambda item: len(item.name))

    return constants

def find_include(block_contents: typ.List[str], include_directive: Directive) -> typ.Optional[str]:
    query = include_directive.value.strip()
    for content in block_contents:
        if query in content:
            return content
    return None

def _indented_include(content, raw_text, include_val) -> str:
    unindented = raw_text.lstrip()
    indent     = raw_text[: -len(unindented)]
    if indent:
        include_val = "\n".join(indent + line for line in include_val.splitlines())
    return content.replace(raw_text, include_val)

# TODO: this should be part of the parsing of all directives
def _parse_prefix(directive: Directive) -> str:
    val = directive.value.strip()
    if val.startswith("'") and val.endswith("'"):
        val = val[1:-1]
    if val.startswith('"') and val.endswith('"'):
        val = val[1:-1]
    return val

def _expand_constants(build_ctx: Context) -> Context:
    constants = _parse_constants(build_ctx)
    # macros have been abandoned for now
    return build_ctx

    done = False
    while not done:
        done         = True
        new_md_files = []
        for md_file in build_ctx.files:
            cur_path = md_file.md_path

            new_elements = list(md_file.elements)

            for block in md_file.blocks:
                new_content = block.content
                for decl_path, name, const_val in constants:
                    namespace   = decl_path.name[: -len(decl_path.suffix)]
                    new_content = new_content.replace(name, const_val)
                    if decl_path == cur_path:
                        new_content = new_content.replace(name, const_val)

                # TODO: There may a bug here that relates to the order
                #   of expanding blocks. Since the addable_val is based
                #   on block.inner_content, there may be issues with
                #   recursive includes

                if new_content == block.content:
                    continue

                done = False

                elem = md_file.elements[block.elem_index]
                new_elements[block.elem_index] = MarkdownElement(
                    elem.md_path, elem.elem_index, elem.md_type, new_content, elem.first_line, None
                )

            new_md_file = MarkdownFile(md_file.md_path, new_elements)
            new_md_files.append(new_md_file)

        build_ctx = Context(new_md_files)

    return build_ctx

def _iter_addable_blocks(md_file: MarkdownFile) -> typ.Iterable[Block]:
    for block in md_file.blocks:
        is_simple_block = not any(
            (
                has_directive(block, 'lp_out'),
                has_directive(block, 'lp_run'),
                has_directive(block, 'lp_make'),
                has_directive(block, 'lp_add'),
                has_directive(block, 'lp_file'),
            )
        )
        if is_simple_block:
            yield block

def _expand_file_add_directives(md_file: MarkdownFile) -> MarkdownFile:
    addable_contents = [b.inner_content for b in _iter_addable_blocks(md_file)]

    new_elements = list(md_file.elements)
    for block in md_file.blocks:
        new_content = block.content
        for lp_add in iter_directives(block, 'lp_add'):
            addable_val = find_include(addable_contents, lp_add)
            if not addable_val:
                continue

            new_content = _indented_include(new_content, lp_add.raw_text, addable_val)

        if new_content == block.content:
            continue

        elem = md_file.elements[block.elem_index]
        new_elements[block.elem_index] = MarkdownElement(
            elem.md_path, elem.elem_index, elem.md_type, new_content, elem.first_line, None
        )

    return MarkdownFile(md_file.md_path, new_elements)

def _iter_expanded_files(md_files: typ.Iterable[MarkdownFile]) -> typ.Iterable[MarkdownFile]:
    # NOTE: To minimize complexity, we don't expand beyond the current
    #   file.
    #
    # This means, if you're looking for what an lp_add
    # directive will expand to, you only have to look for
    # the first occurrence of it's search string in the
    # same file.
    for md_file in md_files:
        prev_md_file = md_file
        while True:
            new_md_file = _expand_file_add_directives(prev_md_file)
            if new_md_file == prev_md_file:
                yield new_md_file
                break
            else:
                prev_md_file = new_md_file

def _iter_block_errors(orig_ctx: Context, build_ctx: Context) -> typ.Iterable[str]:
    """Validate that expansion worked correctly."""
    # NOTE: the main purpose of the orig_ctx is to produce
    #   better error messages. It allows us to point at the
    #   original block, whereas the build_ctx has been
    #   modified and line numbers no longer correspond to
    #   the original file.
    assert len(orig_ctx.files) == len(build_ctx.files)
    for orig_md_file, md_file in zip(orig_ctx.files, build_ctx.files):
        assert orig_md_file.md_path == md_file.md_path
        assert len(orig_md_file.elements) == len(md_file.elements)

        for block in md_file.blocks:
            orig_elem = orig_md_file.elements[block.elem_index]
            for directive in block.directives:
                if directive.name != 'lp_add':
                    continue

                elem = md_file.elements[block.elem_index]

                rel_line_no = 0
                for line in orig_elem.content.splitlines():
                    if directive.raw_text in line:
                        break
                    else:
                        rel_line_no += 1

                line_no = elem.first_line + rel_line_no
                yield f"Error processing {md_file.md_path}"
                yield f"Could not expend '{directive.raw_text}' on line {line_no}"

def _dump_files(build_ctx: Context) -> None:
    for md_file in build_ctx.files:
        for block in md_file.blocks:
            file_directive = get_directive(block, 'lp_file')
            if file_directive is None:
                continue

            path = pl.Path(file_directive.value)
            path.parent.mkdir(parents=True, exist_ok=True)
            with path.open(mode="w", encoding="utf-8") as fh:
                fh.write(block.inner_content)

class SessionBlockOptions(typ.NamedTuple):
    """A Session Block uses either a 'lp_run' or 'lp_out' directive.

    If it is an lp_run directive a session is run and the output
    is captured for later use.
    If it is an lp_out directive, output is either used from
    a previous capture, or if a command is included, then the
    output of the command is captured and immediately used.
    """

    run    : typ.Optional[Directive]
    out    : typ.Optional[Directive]
    command: typ.Optional[str]

    is_stdin_writable: bool
    is_debug         : bool
    keepends         : bool
    timeout          : float
    input_delay      : float
    debug_prefix     : str

    out_fmt : str
    err_fmt : str
    info_fmt: str

    out_prefix: str
    err_prefix: str

def _parse_session_block_options(block: Block) -> typ.Optional[SessionBlockOptions]:
    run_directive = get_directive(block, 'lp_run')
    out_directive = get_directive(block, 'lp_out')
    if run_directive is None and out_directive is None:
        return None

    debug_directive = get_directive(block, 'lp_debug')
    if debug_directive:
        is_debug     = True
        debug_prefix = _parse_prefix(debug_directive) or "> "
    else:
        is_debug     = False
        debug_prefix = "> "

    command: typ.Optional[str]

    is_stdin_writable: bool

    if run_directive:
        command           = run_directive.value
        is_stdin_writable = True
    elif out_directive:
        command           = out_directive.value
        is_stdin_writable = False
    else:
        is_stdin_writable = False
        command           = None

    timeout = get_directive(block, 'lp_timeout')
    if timeout is None:
        timeout_val = 1.0
    else:
        timeout_val = float(timeout.value)

    input_delay = get_directive(block, 'lp_input_delay')
    if input_delay is None:
        input_delay_val = 0.0
    else:
        input_delay_val = float(input_delay.value)

    keepends = True

    out_color = get_directive(block, 'lp_out_color')
    err_color = get_directive(block, 'lp_err_color')

    # TODO: parse out_color
    if err_color is None:
        color_code = TERM_COLORS['red']
        err_fmt    = "\u001b[" + color_code + "m{0}\u001b[0m"
    else:
        color_val = err_color.value.strip()
        if color_val == 'none':
            err_fmt = "{0}"
        elif color_val in TERM_COLORS:
            color_code = TERM_COLORS[color_val]
            err_fmt    = "\u001b[" + color_code + "m{0}\u001b[0m"
        elif COLOR_CODE_RE.match(color_val):
            err_fmt = "\u001b[" + color_val + "m{0}\u001b[0m"
        else:
            valid_codes = ", ".join(sorted(TERM_COLORS.keys()))
            err_msg     = (
                f"Invalid lp_err_color: {color_val}. "
                f"Must be 'none', {valid_codes} or a valid color code."
            )
            raise Exception(err_msg)

    out_prefix = get_directive(block, 'lp_out_prefix')
    err_prefix = get_directive(block, 'lp_err_prefix')

    info_directive = get_directive(block, 'lp_proc_info')
    if info_directive is None:
        info_fmt = "# exit: {exit:>3}"
    else:
        info_fmt = info_directive.value

    return SessionBlockOptions(
        run=run_directive,
        out=out_directive,
        command=command,
        is_stdin_writable=is_stdin_writable,
        is_debug=is_debug,
        keepends=keepends,
        timeout=timeout_val,
        input_delay=input_delay_val,
        debug_prefix=debug_prefix,
        out_fmt="{0}",
        err_fmt=err_fmt,
        info_fmt=info_fmt,
        out_prefix=_parse_prefix(out_prefix) if out_prefix else "",
        err_prefix=_parse_prefix(err_prefix) if err_prefix else "! ",
    )

def _parse_capture_output(capture: Capture, opts: SessionBlockOptions) -> str:
    # TODO: coloring
    # if "\u001b" in capture.stderr:
    #     stderr = capture.stderr
    # elif stderr.strip():
    #     stderr = opts.err_fmt.format(capture.stderr)

    output_lines = []
    for cl in capture.lines:
        if cl.is_err:
            if opts.err_prefix:
                line_val = opts.err_prefix + cl.line
            else:
                line_val = cl.line
        else:
            if opts.out_prefix:
                line_val = opts.out_prefix + cl.line
            else:
                line_val = cl.line
        output_lines.append(line_val.rstrip())

    output = "\n".join(output_lines)

    if not output.endswith("\n"):
        output += "\n"

    if opts.info_fmt != 'none':
        output += opts.info_fmt.format(
            **{
                'exit'   : capture.exit_status,
                'time'   : capture.runtime,
                'time_ms': capture.runtime * 1000,
            }
        )
        output = output.strip()

        # make sure the block closing backticks are
        # on their own line
        if not output.endswith("\n"):
            output += "\n"

    return output

def build(orig_ctx: Context) -> Context:
    # TODO: Immutable datastructures
    #   Context, MarkdownFile
    build_ctx = orig_ctx.copy()

    # TODO: mark build as running
    build_start = time.time()
    # pass 1: expand constants

    build_ctx      = _expand_constants(build_ctx)
    expanded_files = list(_iter_expanded_files(build_ctx.files))
    build_ctx      = Context(expanded_files)

    # phase 3. validate blocks
    error_messages = list(_iter_block_errors(orig_ctx, build_ctx))
    for error_msg in error_messages:
        log.error(error_msg)

    if error_messages:
        sys.exit(1)

    # phase 4. write files with expanded blocks
    #   This has to happen before sub-processes, as those
    #   may use the newly created files.
    _dump_files(build_ctx)

    # TODO: Dependency graph of some kind so we can order
    #   process execution.
    doc_ctx = orig_ctx.copy()

    # phase 5. run sub-processes
    for file_idx, (orig_md_file, md_file) in enumerate(zip(orig_ctx.files, build_ctx.files)):
        captures_by_elem_index: typ.Dict[int, Capture] = {}
        updated_elements      : typ.List[MarkdownElement] = []

        prev_capture_index = -1

        for block in md_file.blocks:
            opts = _parse_session_block_options(block)
            if opts is None:
                continue

            if opts.command:
                log.info(f"  lp_run {opts.command}")

                isession = litprog.session.InteractiveSession(opts.command)

                if opts.is_stdin_writable:
                    stdin_lines = block.inner_content.splitlines(opts.keepends)
                else:
                    stdin_lines = []

                try:
                    for line in stdin_lines:
                        if opts.is_debug:
                            sys.stderr.write(opts.debug_prefix + line.rstrip() + "\n")
                        isession.send(line, delay=opts.input_delay)
                    exit_status = isession.wait(timeout=opts.timeout)
                except Exception:
                    log.error(f"Error processing '{opts.command}'")
                    sys.stdout.write("".join(isession.iter_stdout()))
                    sys.stderr.write("".join(isession.iter_stderr()))
                    raise

                runtime_ms = isession.runtime * 1000
                log.info(f"  lp_run  exit: {exit_status}  time: {runtime_ms:9.3f}ms")

                lines = list(isession.iter_lines())

                # TODO: limit output using lp_max_bytes and lp_max_lines
                # TODO: output escaping/fence style change and errors

                prev_capture_index = block.elem_index
                captures_by_elem_index[prev_capture_index] = Capture(
                    opts.command, exit_status, isession.runtime, lines
                )

            if opts.out:
                if prev_capture_index < 0:
                    output = "<invalid no output captured>\n"
                else:
                    # NOTE: The capture may be the same as the elem_index
                    #   of the current block.
                    capture            = captures_by_elem_index[prev_capture_index]
                    prev_capture_index = -1
                    output             = _parse_capture_output(capture, opts)

                elem = md_file.elements[block.elem_index]
                assert elem.md_type == 'block'

                header_lines = [
                    line
                    for line in elem.content.splitlines(opts.keepends)
                    if line.startswith("```") or line.startswith("# lp_")
                ]

                last_line   = header_lines.pop()
                new_content = "".join(header_lines) + output + last_line

                if elem.content != new_content:
                    updated_elem = MarkdownElement(
                        elem.md_path,
                        elem.elem_index,
                        elem.md_type,
                        new_content,
                        elem.first_line,
                        None,
                    )
                    updated_elements.append(updated_elem)

        # phase 6. rewrite output blocks
        if not any(updated_elements):
            continue

        new_elements = list(orig_md_file.elements)
        for elem in updated_elements:
            orig_elem = orig_md_file.elements[elem.elem_index]
            assert "lp_out" in orig_elem.content
            new_elements[elem.elem_index] = elem

        new_md_file      = MarkdownFile(md_file.md_path, new_elements)
        new_file_content = str(new_md_file)
        with new_md_file.md_path.open(mode="w", encoding="utf-8") as fh:
            fh.write(new_file_content)
        log.info(f"Updated {new_md_file.md_path}")
        doc_ctx.files[file_idx] = new_md_file

    return doc_ctx


from twisted.internet import reactor
from twisted.internet import defer
from twisted.internet import task
from kademlia.network import Server
from kademlia.log import Logger
import time
import copy
import sys
import signal
from twisted.python import log

log.startLogging(sys.stdout)

# time PYTHONPATH=. python examples/benbenchmark.py

class BenchmarkClient(object):
    def __init__(self, server, loops):
        self.verified_values = 0
        self.key_vals_to_store = [("a key %d" % i, "some value %s" % i) for i in range(0, loops)]
        self.failed_value_names = []
        self.server = server
        self.loops = loops
        self.log = Logger(system=self)

    @defer.inlineCallbacks
    def store_many(self, kv_list):
        """Stores all value v against key k for k, v in kv_list """
        self.log.debug("Attempting to store %s (key, val) pairs" % len(kv_list))
        dfrs = []
        for k, v in kv_list:
            dfrs.append(self.write_key_val(k, v))
        dl =  defer.DeferredList(dfrs, consumeErrors=True)
        res = yield dl
        writes_dispatched = len([succeeded  for succeeded, result in res if succeeded])
        write_failure_results = [result  for _succeeded, result in res if not succeeded]
        self.log.debug("%s write failures : %s" % (len(write_failure_results), write_failure_results))
        defer.returnValue(writes_dispatched)

    @defer.inlineCallbacks
    def read_and_verify_many(self, kv_list):
        """Attempts to read all v stored against key k for k, v in kv_list - and
        keeps score if expected values are missing."""
        self.log.debug("Attempting to read and verify %s (key, val) pairs" % len(kv_list))
        dfrs = []
        self.verified_values = 0
        self.failed_value_names = []
        for k, v in kv_list:
            dfrs.append(self.read_key_val(k, v))
        dl =  defer.DeferredList(dfrs, consumeErrors=True)
        res = yield dl
        read_requests_dispatched = len([succeeded  for succeeded, result in res if succeeded])
        defer.returnValue(read_requests_dispatched)

    @defer.inlineCallbacks
    def retry_writes_and_reads(self):
        for retry in range(1, 6):
            yield self._sleep(6)
            self.log.debug("***************** Retrying failures ******************")
            self.log.debug("Retry %s: Failed to retrieve previously stored values for %s items: %s " % (retry, len(self.failed_value_names), self.failed_value_names))
            # self.log.debug("***************** Pinging ******************")
            # Ugly ugly hack
            #pingres = yield self.server.protocol.ping(('127.0.0.1', 8468), self.server.protocol.sourceNode.id)
            # self.log.debug("ping sent... : %s " % str(pingres))
            # bootstrapres = yield self.server.bootstrap([('127.0.0.1', 8468)])
            # self.log.debug("bootstrap sent... : %s " % str(bootstrapres))

            yield self._sleep(6)
            self.log.debug("***************** Retrying writes which failed verification ******************")
            failed_value_names_copy = copy.copy(self.failed_value_names)
            writes_dispatched = yield self.store_many(failed_value_names_copy)
            self.log.debug("Successfully dispatched %s (repeat)  write requests" % writes_dispatched)

            yield self._sleep(5)
            self.log.debug("***************** Retrying read ******************")
            read_requests_dispatched = yield self.read_and_verify_many(failed_value_names_copy)

            self.log.debug("Successfully dispatched %s (repeated) read requests" % read_requests_dispatched)
            self.log.debug("***************** Retrying summary ******************")
            self.log.debug("After write/read retry cycle %s, still failed to retrieve previously stored values for %s items: %s " % (retry, len(self.failed_value_names), self.failed_value_names))

            #self.log.debug("Fetching dump....")
            #dumpres = yield self.server.dump()
            #self.log.debug("Finally, dump result is : %s" %(dumpres,))

    @defer.inlineCallbacks
    def benchmark(self, _result):
        bw = time.time()
        writes_dispatched = yield self.store_many(self.key_vals_to_store)
        aw = time.time()
        self.log.debug("Successfully dispatched %s write requests" % writes_dispatched)

        read_requests_dispatched = yield self.read_and_verify_many(self.key_vals_to_store)
        ar = time.time()
        self.log.debug("Successfully dispatched %s read requests" % read_requests_dispatched)
        self.log.debug("%s of %s writes dispatched; took %s seconds = %s ops/sec" % (writes_dispatched, self.loops, aw - bw,  writes_dispatched / (1.0 * aw - bw)  ))
        self.log.debug("%s of %s reads dispatched; took %s seconds = %s ops/sec  (verified values: %s; failed values %s)" % (read_requests_dispatched, self.loops, ar - aw, self.loops / (1.0 * ar - aw), self.verified_values, len(self.failed_value_names)))

        if self.failed_value_names:
            yield self.retry_writes_and_reads()
        else:
            self.log.debug("All values verified successfully")
        self.log.debug("In the end %s values stored on this client node" % len(self.server.storage.data))

        reactor.stop()

    @defer.inlineCallbacks
    def write_key_val(self, key, val):
        yield self.server.set(key, val)

    @defer.inlineCallbacks
    def read_key_val(self, key, expected_value):
        result = yield self.server.get(key)
        if result == expected_value:
            self.verified_values = self.verified_values + 1
        else:
            self.log.debug( "Unexpected value read(k='%s'): got v='%s'; expected='%s'" % (key, result, expected_value))
            self.failed_value_names.append((key, expected_value))
        defer.returnValue(result)

    @defer.inlineCallbacks
    def _sleep(self, seconds):
        yield task.deferLater(reactor, seconds, lambda: True)

if __name__ == '__main__':
    n_transactions = 5
    if len(sys.argv) > 1:
        n_transactions = int(sys.argv[1])

    def signal_handler(signal, frame):
        print('You pressed Ctrl+C!')
        reactor.callFromThread(reactor.stop)
    signal.signal(signal.SIGINT, signal_handler)
    server = Server()
    client = BenchmarkClient(server, n_transactions)
    server.listen(8880)
    bootstrap_addrs = [("127.0.0.1", port ) for port in [8881, 8882, 8883]]

    server.bootstrap(bootstrap_addrs, fixed_supernode_addrs=bootstrap_addrs).addCallback(client.benchmark)
    reactor.run()


import csv
import numpy as np
from sklearn import linear_model
from sklearn.metrics.classification import accuracy_score
from sklearn.metrics import classification_report

def loaddata(filename,instanceCol):
    file_reader = csv.reader(open(filename,'r'),delimiter=',')
    x = []
    y = []
    for row in file_reader:
        x.append(row[0:instanceCol])
        y.append(row[-1])
    return np.array(x[1:]).astype((np.float32)), np.array(y[1:]).astype(np.int)

def fractal_modeldata(filename, inc):
    print(filename)
    X, Y = loaddata(filename, inc)
    np.random.seed(13)
    indices = np.random.permutation(2038)
    test_size = int(0.3 * len(indices))
    X_train = X[indices[:-test_size]]
    Y_train = Y[indices[:-test_size]]
    X_test = X[indices[-test_size:]]
    Y_test = Y[indices[-test_size:]]
    classifier = linear_model.SGDClassifier(max_iter=10, loss='log')
    classifier.fit(X_train, Y_train)
    Y_pred = classifier.predict_proba(X_test)
    print(accuracy_score(Y_test, Y_pred)*100)
    print(classification_report(Y_test, Y_pred))

if __name__ == '__main__':
    fractal_modeldata('D:\\Databases\\Steganalysis\\Dataset\\LogFBank\\LogFBank-Features-hide4pgp-100.csv', 27)
    fractal_modeldata('D:\\Databases\\Steganalysis\\Dataset\\FBank\\FBank-Features-hide4pgp-100.csv', 27)
    fractal_modeldata('D:\\Databases\\Steganalysis\\Dataset\\Fractal\\Fractal-Features-hide4pgp-100.csv', 27)
    fractal_modeldata('D:\\Databases\\Steganalysis\\Dataset\\MFCC\\MFCC-Features-hide4pgp-100.csv', 27)
    fractal_modeldata('D:\\Databases\\Steganalysis\\Dataset\\LPC\\lpc-Features-hide4pgp-100.csv', 27)
    fractal_modeldata('D:\\Databases\\Steganalysis\\Dataset\\DeltaMFCC\\deltaMFCC-Features-hide4pgp-100.csv', 15)


import sys
from PyQt5.QtWidgets import QFileDialog
import tester
from PyQt5 import QtGui, QtWidgets, QtCore
import serial
import glob
from interface import Listener_Thread
import pandas as pd

# noinspection PyPep8Naming
class Tester_Mainwindow(QtWidgets.QDialog, tester.Ui_Form):
    def __init__(self, parent=None):
        super(Tester_Mainwindow, self).__init__(parent)
        self.setupUi(self)

        # Set Fixed Size W:250 H:230
        self.setFixedSize(250, 230)

        # Pre
        self.PreFeed = 100

        # Serial Port
        serial_ports_list = self.list_serial_ports()
        if not serial_ports_list:
            print("No Ports Available")
        else:
            self._Port = Listener_Thread(serial_ports_list[0], read_timeout=7, write_timeout=7)
            self._Port.start()

        # Connections
        # Buttons
        self.pushButtonDown.clicked.connect(self.pushButtonDown_Clicked)
        self.pushButtonUp.clicked.connect(self.pushButtonUp_Clicked)
        self.pushButtonLeft.clicked.connect(self.pushButtonLeft_Clicked)
        self.pushButtonRight.clicked.connect(self.pushButtonDown_Clicked)
        self.pushButtonSend.clicked.connect(self.pushButtonSend_Clicked)
        self.pushButtonSpindel.clicked.connect(self.pushButtonSpindel_Clicked)
        self.pushButtonPumpe.clicked.connect(self.pushButtonPumpe_Clicked)
        self.pushButtonStop.clicked.connect(self.pushButtonStop_Clicked)
        self.pushButtonExcel.clicked.connect(self.pushButtonExcel_Clicked)

    def pushButtonExcel_Clicked(self):
        fname = QFileDialog.getOpenFileName(self, 'Open file', 'C:\\Users\\sebas\\source\\repos\\Marvin-PC-Interface', "CSV (*.csv);;Text (*.txt)")
        data = []

        try:
            data = pd.read_csv(fname[0])
        except:
            print("Reading File was not successful")

        if "csv" in fname[0]:
            tmp = "XYF" + ";" + str(data["X"][0]) + ";" + str(data["Y"][0]) + ";" + str(data["F"][0]) + "@"
            self._Port.sendMessage(tmp)
            tmp = "XYF" + ";" + str(data["X"][1 + 1]) + ";" + str(data["Y"][1 + 1]) + ";" + str(data["F"][1 + 1]) + "@"
            self._Port.sendMessage(tmp)
            tmp = "XYF" + ";" + str(data["X"][2 + 2]) + ";" + str(data["Y"][2 + 2]) + ";" + str(data["F"][2 + 2]) + "@"
            self._Port.sendMessage(tmp)
            r = str(self._Port.getMessage())
            count = 0
            while count >= 3:
                r = str(self._Port.getMessage())
                if "ACK" not in r:
                    pass
                else:
                    print(r)
                    count = count + 1

            for i in range(3, data.shape[0]):
                tmp = "XYF" + ";" + str(data["X"][i]) + ";" + str(data["Y"][i]) + ";" + str(data["F"][i]) + "@"
                self._Port.sendMessage(tmp)

                r = str(self._Port.getMessage())
                count = 0
                while True:
                    r = str(self._Port.getMessage())
                    if "ACK" not in r:
                        pass
                    else:
                        print(r)
                        print(i)
                        break
        else:
            with open(fname[0]) as f:
                content = f.readlines()
            for i in range(0, len(content)):
                content[i] = content[i].strip()
                tmp = content[i];
                self._Port.flushReceiveBuffer()
                self._Port.sendMessage(tmp)
                r = str(self._Port.getMessage())
                dirtyhackflag = 0
                while True:
                    if "XYF" in tmp:
                        if "Reached X" in r:
                            dirtyhackflag = dirtyhackflag + 1
                            if dirtyhackflag >=2:
                                print(r)
                                break
                    else:
                        if "ACK" in r:
                            break
                    print(r)
                    r = str(self._Port.getMessage())
            # tmp = content[0] + "@"
            # self._Port.sendMessage(tmp)
            # tmp = content[1] + "@"
            # self._Port.sendMessage(tmp)
            # tmp = content[2] + "@"
            # self._Port.sendMessage(tmp)
            # self._Port.flushReceiveBuffer()
            # r = str(self._Port.getMessage())
            # count = 0
            # while count >= 3:
            #     r = str(self._Port.getMessage())
            #     if "ACK" not in r:
            #         pass
            #     else:
            #         print(r)
            #         count = count + 1
            #
            # for i in range(3, data.shape[0]+1):
            #     tmp = content[i] + "@"
            #     self._Port.sendMessage(tmp)
            #
            #     r = str(self._Port.getMessage())
            #     count = 0
            #     while True:
            #         r = str(self._Port.getMessage())
            #         if ("Reached" in r) or ("ACK" in r and "XYF" not in r):
            #             print(r)
            #             print(i)
            #             break

    def pushButtonStop_Clicked(self):
        tmp = "STOP@"
        self._Port.sendMessage(tmp)

    def pushButtonSpindel_Clicked(self):
        tmp = "S;" + str(self.lineEditSpindel.text()) + "@"
        self._Port.sendMessage(tmp)

    def pushButtonPumpe_Clicked(self):
        tmp = "P;" + str(self.lineEditPolierpumpe.text()) + "@"
        self._Port.sendMessage(tmp)

    def pushButtonDown_Clicked(self):
        self._Port.sendMessage("XYF;0;1;100@")

    def pushButtonUp_Clicked(self):
        self._Port.sendMessage("XYF;0;-1;100@")

    def pushButtonLeft_Clicked(self):
        self._Port.sendMessage("XYF;-1;0;100@")

    def pushButtonRight_Clicked(self):
        self._Port.sendMessage("XYF;1;0;100@")

    def pushButtonSend_Clicked(self):
        if any(char.isdigit() for char in self.lineEditX.text()):
            x = self.lineEditX.text()
        if any(char.isdigit() for char in self.lineEditY.text()):
            y = self.lineEditY.text()
        if any(char.isdigit() for char in self.lineEditFeed.text()):
            feed = self.lineEditFeed.text()
        self._Port.sendMessage("XYF;" + x + ";" + y + ";" + feed + "@")

    def list_serial_ports(self):
        if sys.platform.startswith('win'):
            ports = ['COM%s' % (i + 1) for i in range(256)]
        elif sys.platform.startswith('linux') or sys.platform.startswith('cygwin'):
            # this excludes your current terminal "/dev/tty"
            ports = glob.glob('/dev/tty[A-Za-z]*')
        elif sys.platform.startswith('darwin'):
            ports = glob.glob('/dev/tty.*')
        else:
            raise EnvironmentError('Unsupported platform')

        result = []
        for port in ports:
            try:
                s = serial.Serial(port)
                s.close()
                result.append(port)
            except (OSError, serial.SerialException):
                pass
        return result


import cv2, glob, os
import matplotlib.pyplot as plt
import xlrd
import xlsxwriter

path = 'dataset_label.xlsx'
wb = xlrd.open_workbook(path)
sheet = wb.sheet_by_index(3)

coor = -1
index = 5
for i in range(sheet.nrows):
    coor0 = sheet.cell_value(i, 0)
    if coor0 == 'Image':
        continue
    # if coor0 != 'scene'+str(index):
    #     continue
    # if coor0 == 'scene'+str(index+2):
    #     break
    # coor1 = int(sheet.cell_value(i, 1))
    coor0 = int(sheet.cell_value(i, 0))
    x = int(sheet.cell_value(i, 1))
    y = int(sheet.cell_value(i, 2))
    w = int(sheet.cell_value(i, 3))
    h = int(sheet.cell_value(i, 4))
    #path0 = '/home/ziyi/Documents/test_label/test/Scene'+str(index)+'/Delete/Delete'+str(index)+'_One-Face' + str(coor1) + '.png'
    # path0 = '/home/ziyi/Documents/test_label/test/Scene' + str(index) + '/Delete/Delete' + str(coor1) + '.png'
    imagePath = '/home/ziyi/Documents/test_label/Delete/Delete' + str(coor0) + '.png'
    path0 = '/home/ziyi/Documents/test_label/test/Delete/Delete' + str(coor0) + '.png'
    # if coor == coor1:
    #     imagePath = path0
    # else:
    #     imagePath = '/home/ziyi/Documents/test_label/Scene'+str(index)+'/Delete/Delete'+str(coor1)+'.png'

    image = cv2.imread(imagePath)
    cv2.rectangle(image, (x, y), (x + w, y + h), (0, 255, 0), 2)
    cv2.imwrite(path0, image)
    # coor = coor1


"""tests cogeo_mosaic.overview."""

import json
import os

import rasterio
from click.testing import CliRunner
from rio_cogeo.cogeo import cog_validate
from rio_cogeo.profiles import cog_profiles

from cogeo_mosaic.mosaic import MosaicJSON
from cogeo_mosaic.overviews import create_overview_cogs

asset1 = os.path.join(os.path.dirname(__file__), "fixtures", "cog1.tif")
asset2 = os.path.join(os.path.dirname(__file__), "fixtures", "cog2.tif")
assets = [asset1, asset2]
mosaic_content = MosaicJSON.from_urls(assets).dict(exclude_none=True)

deflate_profile = cog_profiles.get("deflate")
deflate_profile.update({"blockxsize": 256, "blockysize": 256})

def test_overview_valid():
    """Should work as expected (create cogeo file)."""
    config = {
        "GDAL_DISABLE_READDIR_ON_OPEN": "TRUE",
        "GDAL_TIFF_INTERNAL_MASK": "TRUE",
        "GDAL_TIFF_OVR_BLOCKSIZE": "128",
    }
    runner = CliRunner()
    with runner.isolated_filesystem():
        with open("mosaic.json", "w") as f:
            f.write(json.dumps(mosaic_content))

        create_overview_cogs("mosaic.json", deflate_profile, config=config, threads=1)

        assert cog_validate("mosaic_ovr_0.tif")

        with rasterio.open("mosaic_ovr_0.tif") as src:
            assert src.height == 512
            assert src.width == 512
            assert src.meta["dtype"] == "uint16"
            assert src.is_tiled
            assert src.profile["blockxsize"] == 256
            assert src.profile["blockysize"] == 256
            assert src.compression.value == "DEFLATE"
            assert src.interleaving.value == "PIXEL"
            assert src.overviews(1) == [2]

        with rasterio.open("mosaic_ovr_0.tif", OVERVIEW_LEVEL=0) as src:
            assert src.block_shapes[0] == (128, 128)


from django.utils.deprecation import MiddlewareMixin
from django.http import HttpResponse

class MiddleWareTest(MiddlewareMixin):
    def process_request(self,request):
        rep_ip = request.META.get("REMOTE_ADDR")
        print(rep_ip)
        if rep_ip == "10.10.92.38":  ## 返回响应
            return HttpResponse("同学，请绕路，不支持你访 问！！！")
        print('process_request')

    def process_view(self,request,callback,callbackargs,callbackkwargs):

        version=callbackkwargs.get("date")
        if version == "v1":
            return HttpResponse("版本过期")

        print('process_view')

    def process_exception(self,request,exception):
        print('process_exception')
        from Qshop.settings import BASE_DIR
        import os
        import time
        file=os.path.join(BASE_DIR,'error.log')
        now_time=time.strftime('%Y-%m-%d %H:%M:%S',time.localtime())
        with open(file,'a') as f:
            content="[%s]:%s \n" %(now_time,str(exception))
            f.write(content)
        pass

    def process_template_response(self,request,response):
        print('process_template_response')
        return response

    def process_response(self,request,response):
        print('process_response')
        print(response)
        return response


'''
Python tools for creating and parsing AXI communications.
'''

import asyncio
import random
import logging
import time

from pyvivado import signal

logger = logging.getLogger(__name__)

# `Comm` objects are registered here by the name of the module they are
# responsible for communicating with.
module_register = {}

# Response code for AXI.
OKAY = 0
EXOKAY = 1
SLVERR = 2
DECERR = 3

# Define a record type for Master-to-Slave AXI
axi4lite_m2s_type = signal.Record(
    contained_types=(
        ('araddr', signal.StdLogicVector(width=32)),
        ('arprot', signal.StdLogicVector(width=3)),
        ('arvalid', signal.std_logic_type),
        ('awaddr', signal.StdLogicVector(width=32)),
        ('awprot', signal.StdLogicVector(width=3)),
        ('awvalid', signal.std_logic_type),
        ('bready', signal.std_logic_type),
        ('rready', signal.std_logic_type),
        ('wdata', signal.StdLogicVector(width=32)),
        ('wstrb', signal.StdLogicVector(width=4)),
        ('wvalid', signal.std_logic_type),
    ),
    name='axi4lite_m2s',
)

# Define a record type for Slave-to-Master AXI
axi4lite_s2m_type = signal.Record(
    contained_types=(
        ('arready', signal.std_logic_type),
        ('awready', signal.std_logic_type),
        ('bresp', signal.StdLogicVector(width=2)),
        ('bvalid', signal.std_logic_type),
        ('rdata', signal.StdLogicVector(width=32)),
        ('rresp', signal.StdLogicVector(width=2)),
        ('rvalid', signal.std_logic_type),
        ('wready', signal.std_logic_type),
    ),
    name='axi4lite_s2m',
)

def make_empty_axi4lite_m2s_dict():
    '''
    Creates and empty master-to-slave AXI dictionary.
    '''
    return {
        'araddr': 0,
        'arprot': None,
        'arvalid': 0,
        'awaddr': 0,
        'awprot': None,
        'awvalid': 0,
        'bready': 0,
        'rready': 0,
        'wdata': 0,
        'wstrb': None,
        'wvalid': 0,
    }

def make_empty_axi4lite_s2m_dict():
    '''
    Creates and empty slave-to-master AXI dictionary.
    '''
    return {
        'arready': 1,
        'awready': 1,
        'bresp': 0,
        'bvalid': 0,
        'rdata': 0,
        'rresp': 0,
        'rvalid': 0,
        'wready': 1,
    }

# Used to define whether AXI command are reading or writing.
READ_TYPE = 'READ'
WRITE_TYPE = 'WRITE'

class ConnCommandHandler(object):
    '''
    This handler receives `CommCommand` objects and sends their AXI
    commands over the passed in `Connection`.  The responses are
    then processed by the `CommCommand` object that send them.

    This handler is useful to simplify communication with the FPGA.
    '''

    def __init__(self, conn):
        '''
        `conn`: A `Connection` object that the handler uses to communicate
                with the FPGA.
        '''
        self.conn = conn

    def send(self, commands):
        '''
        Sends a list of `commands` where are CommCommand objects to the FPGA
        and processes the responses.
        '''
        for command in commands:
            if isinstance(command, FakeWaitCommand):
                time.sleep(command.sleep_time)
            rs = []
            for ac in command.axi_commands:
                assert(ac.readorwrite in (WRITE_TYPE, READ_TYPE))
                if ac.readorwrite == WRITE_TYPE:
                    if ac.constant_address:
                        r = self.conn.write_repeat(address=ac.start_address,
                                              data=ac.data)
                    else:
                        r = self.conn.write(address=ac.start_address, data=ac.data)
                else:
                    if ac.constant_address:
                        raise Exception('Reading from constant address not supported yet.')
                    else:
                        r = self.conn.read(address=ac.start_address, length=ac.length)
                rs.append(r)
            command.process_response((None, rs))

class DictCommandHandler(object):
    '''
    This handler receives `CommCommand` objects and stores them.
    When the `make_command_dicts` method is called the handler returns
    a list of AXI master-to-slave dictionaries that specify the commands.
    It can also parse the output AXI slave-to-master dictionaries from
    a simulation.

    This handler is useful to fake communication when running simulations.
    '''

    def __init__(self):
        self.unsent_commands = []
        self.sent_commands = []

    def send(self, commands):
        self.unsent_commands += commands

    def make_command_dicts(self):
        '''
        Generates slave-to-master AXI dictionaries from the CommCommands
        that have been given to the handler.  These dictionaries can
        be passed as input to simluations.
        '''
        ds = []
        while self.unsent_commands:
            command = self.unsent_commands.pop(0)
            if isinstance(command, FakeWaitCommand):
                for i in range(command.clock_cycles):
                    ds.append(make_empty_axi4lite_m2s_dict())
            for ac in command.axi_commands:
                for index in range(ac.length):
                    d = make_empty_axi4lite_m2s_dict()
                    if ac.constant_address:
                        address = ac.start_address
                    else:
                        address = ac.start_address + index
                    if ac.readorwrite == READ_TYPE:
                        d['araddr'] = address
                        d['arvalid'] = 1
                    else:
                        d['awaddr'] = address
                        d['awvalid'] = 1
                        d['wvalid'] = 1
                        d['wdata'] = ac.data[index]
                    ds.append(d)
            self.sent_commands.append(command)
        return ds

    def consume_response_dicts(self, ds):
        '''
        Takes a list of slave-to-master dictionaries obtained from
        a simulation and processes them using the 'CommCommand' objects
        that sent them.

        If their was a bug in the AXI communication (i.e. no response
        when their should have been), then things can get out of sync,
        and the wrong 'CommCommand' objects will process the wrong responses.
        '''
        for command in self.sent_commands:
            results = []
            first_e = None
            for ac in command.axi_commands:
                error_help = ' Command description: {}'.format(ac.description)
                bad_response = False
                rs = []
                e = None
                for index in range(ac.length):
                    r = None
                    while r is None:
                        if len(ds) == 0:
                            raise Exception('DictCommandHandler is out of sync.  Probably a bug in the AXI communication.' + error_help)
                        d = ds.pop(0)
                        r = None
                        if ac.readorwrite == READ_TYPE:
                            if d['rvalid']:
                                r = (d['rresp'], d['rdata'])
                        elif ac.readorwrite == WRITE_TYPE:
                            if d['bvalid']:
                                r = (d['bresp'], None)
                    if r[0] != OKAY:
                        bad_response = True
                    rs.append(r)
                if ac.readorwrite == WRITE_TYPE:
                    if len(rs) != ac.length:
                        e = Exception('Incorrect number of response.' + error_help)
                    elif bad_response:
                        e = Exception('Received a bad response.' + error_help)
                    result = None
                else:
                    if bad_response:
                        e = Exception('Received a bad response.' + error_help)
                        result = None
                    else:
                        result = [r[1] for r in rs]
                if first_e is None and e is not None:
                    first_e = e
                results.append(result)
            command.process_response((first_e, results))

class AxiCommand(object):
    '''
    Defines a series AXI4Lite master-to-slave commands.
    '''

    def __init__(self, start_address, length, readorwrite, data=None,
                 constant_address=False, description=None):
        '''
        `start_address`: The address on which the first AXI command operates.
        `constant_address`: If this is `True` we keep operating on the same
             address, otherwise we increment.
        `length`: The number of commands.
        `readorwrite`: Can be either `READ_TYPE` or `WRITE_TYPE`.
        'data': A list of integers to send (if it is a write command).
        `description`: An optional description for debugging purposes.
        '''
        max_address = pow(2, 32-1)
        self.start_address = start_address
        self.length = length
        self.readorwrite = readorwrite
        self.constant_address = constant_address
        self.description = description
        assert(readorwrite in (READ_TYPE, WRITE_TYPE))
        self.data = data
        if readorwrite == READ_TYPE:
            assert(self.data is None)
        else:
            assert(len(self.data) == length)
        assert(start_address <= max_address)
        if not constant_address:
            assert(start_address + length-1 <= max_address)

class CommCommand(object):
    '''
    Defines a list of `AXICommand`s along with methods to
    parse the response.
    '''

    def __init__(self, description=None):
        # The futures attibute is populated with the result when
        # it is processed.
        # See asynio for details on futures.  But basically
        # future.done() tells you if it is finished and
        # future.result() will give you the result.

        # We use futures here so that we can create and send commands
        # but not expect to receive results immediately.  This is
        # particularly useful during simulation when we create and
        # send all the commands before the simulation starts. But
        # obviously can't process the responses until after the
        # simulation is finished.
        self.future = asyncio.Future()
        self.description = description

    def process_result(self, result):
        '''
        Processes the results from the AXI commands and returns
        an (Exception, result) Tuple.
        '''
        return None, result

    def process_response(self, response):
        '''
        Takes an (Exception, result) tuple and sets the exception
        and result on the future.
        '''
        e, result = response
        if e is not None:
            self.future.set_exception(e)
        else:
            e, processed_result = self.process_result(result)
            if e is not None:
                self.future.set_exception(e)
            else:
                self.future.set_result(processed_result)

    def set_unsigneds_commands(
            self, values, address, description=None, constant_address=False):
        '''
        Create `AxiCommand`s for writing unsigned integers.
        '''
        for value in values:
            assert(value < pow(2, 32))
        command = AxiCommand(
            start_address=address,
            length=len(values),
            readorwrite=WRITE_TYPE,
            data=values,
            constant_address=constant_address,
            description=description,
        )
        return [command]

    def set_signeds_commands(
            self, values, address, constant_address=False, description=None):
        '''
        Create `AxiCommand`s for writing signed integers.
        '''
        offset = pow(2, 32)
        unsigneds = [v+offset if v < 0 else v for v in values]
        return self.set_unsigneds_commands(
            unsigneds, address, constant_address, description=description)

    def get_unsigneds_commands(
            self, address, length=1, constant_address=False, description=None):
        '''
        Create `AxiCommand`s for reading unsigned integers.
        '''
        command = AxiCommand(
            start_address=address,
            length=length,
            readorwrite=READ_TYPE,
            constant_address=constant_address,
            description=description,
        )
        return [command]

    def set_unsigned_commands(self, value, address, description=None):
        '''
        Create `AxiCommand`s for writing an unsigned integer.
        '''
        return self.set_unsigneds_commands(
            values=[value], address=address, description=description)

    def set_signed_commands(
            self, value, address, constant_address=False, description=None):
        '''
        Create `AxiCommand`s for writing signed integers.
        '''
        if value < 0:
            value += pow(2, 32)
        return self.set_unsigned_commands(
            value, address, description=description)

    def trigger_commands(self, address, description=None):
        '''
        Create `AxiCommand`s for writing a 0 to an address.  This
        is used as a trigger sometimes.
        '''
        return self.set_unsigned_commands(0, address, description=description)

    def get_unsigned_commands(self, address, description=None):
        '''
        Create `AxiCommand`s for reading an unsigned integer.
        '''
        return self.get_unsigneds_commands(
            address, length=1, description=description)

    def get_boolean_commands(self, address, description=None):
        '''
        Create `AxiCommand`s for reading a boolean.
        '''
        command = AxiCommand(
            start_address=address,
            length=1,
            readorwrite=READ_TYPE,
            description=description)
        return [command]

    def process_get_boolean(self, result):
        '''
        Process a response from reading a boolean.
        '''
        e = None
        r = None
        if result == 1:
            r = True
        elif result == 0:
            r = False
        else:
            r = None
            e = Exception('Unknown return value ({}). Command Description: {}'.format(result, self.description))
        return e, r

    def set_boolean_commands(self, value, address, description=None):
        '''
        Create `AxiCommand`s for writing a boolean.
        '''
        assert(value in (True, False))
        if value:
            data = [1]
        else:
            data = [0]
        command = AxiCommand(
            start_address=address,
            length=1,
            readorwrite=WRITE_TYPE,
            data=data,
            description=description,
        )
        return [command]

class FakeWaitCommand(CommCommand):
    '''
    This is used when we're sending commands to a simulation.
    The `DictCommandHandler` translates it into a bunch of empty
    master-to-slave dictionaries.
    It can be also used when communicating with an FPGA to indicate
    a sleep time.  This is for compatibility of tests.
    '''

    def __init__(self, clock_cycles, sleep_time=0, description=None):
        super().__init__(description=description)
        self.clock_cycles = clock_cycles
        self.axi_commands = []
        self.sleep_time = sleep_time

class GetBooleanCommand(CommCommand):

    def __init__(self, address, description=None):
        super().__init__(description=description)
        self.axi_commands = self.get_boolean_commands(
            address=address, description=description)

    def process_result(self, result):
        return self.process_get_boolean(result[0][0])

class SetBooleanCommand(CommCommand):

    def __init__(self, value, address, description=None):
        super().__init__(description=description)
        self.axi_commands = self.set_boolean_commands(
            value=value, address=address, description=description)

class GetUnsignedCommand(CommCommand):

    def __init__(self, address, description=None):
        super().__init__(description=description)
        self.axi_commands = self.get_unsigned_commands(
            address=address, description=description)

    def process_result(self, result):
        return None, result[0][0]

class SetUnsignedCommand(CommCommand):

    def __init__(self, value, address, description=None):
        super().__init__(description=description)
        self.axi_commands = self.set_unsigned_commands(
            value=value, address=address, description=description)

class SetUnsignedsCommand(CommCommand):

    def __init__(self, values, address, constant_address=False, description=None):
        super().__init__(description=description)
        self.axi_commands = self.set_unsigneds_commands(
            values=values, address=address, constant_address=constant_address,
            description=description)

class TriggerCommand(CommCommand):

    def __init__(self, address, description=None):
        super().__init__(description=description)
        self.axi_commands = self.trigger_commands(
            address, description=description)

class CombinedCommands(CommCommand):
    '''
    Collects a number of `CommCommand` objects into a single one.

    The result returns is a list of the results of the contained
    `CommCommand` objects.
    '''

    def __init__(self, commands, description=None):
        super().__init__(description=description)
        self.subcommands = commands
        self.axi_commands = []
        for sc in self.subcommands:
            self.axi_commands += sc.axi_commands

    def process_result(self, result):
        new_result = []
        e = None
        for sc in self.subcommands:
            rs = []
            for ac in sc.axi_commands:
                rs.append(result.pop(0))
            sub_e, sub_r = sc.process_result(rs)
            if sub_e is not None and e is None:
                e = sub_e
            new_result.append(sub_r)
        return e, new_result

class Comm(object):
    '''
    Subclasses of this create python interfaces for specific module
    that create `CommCommand`s to send to a communications handler.
    '''

    def fake_wait(self, clock_cycles, sleep_time=0):
        command = FakeWaitCommand(
            clock_cycles=clock_cycles, sleep_time=sleep_time)
        self.handler.send([command])
        return command.future

    def get_boolean(self, address, description=None):
        command = GetBooleanCommand(address=address, description=description)
        self.handler.send([command])
        return command.future

    def set_boolean(self, value, address, description=None):
        command = SetBooleanCommand(
            value=value, address=address, description=description)
        self.handler.send([command])
        return command.future

    def get_unsigned(self, address, description=None):
        command = GetUnsignedCommand(address=address, description=description)
        self.handler.send([command])
        return command.future

    def set_unsigned(self, value, address, description=None):
        command = SetUnsignedCommand(value=value, address=address,
                                     description=description)
        self.handler.send([command])
        return command.future

    def set_unsigneds(
            self, values, address, constant_address=False, description=None):
        command = SetUnsignedsCommand(
            values=values, address=address, constant_address=constant_address,
            description=description)
        self.handler.send([command])
        return command.future

    def trigger(self, address, description=None):
        command = TriggerCommand(address=address, description=description)
        self.handler.send([command])
        return command.future

class AxiDummy(object):
    '''
    Dummy axi module for using to create tests.
    '''

    MAX_PIPE_LENGTH = 10

    def __init__(self, max_pipe_length=1):
        self.reset()
        self.max_pipe_length = max_pipe_length

    def reset(self):
        self.write_counter = 0
        self.read_counter = 0

    def handle_write(self):
        logger.debug('writing')
        if self.write_counter:
            raise Exception('Received write before finished previous write.')
        self.write_counter = random.randint(2, self.max_pipe_length+1)

    def handle_read(self):
        if self.read_counter:
            raise Exception('Received read before finished previous read.')
        self.read_counter = random.randint(2, self.max_pipe_length)

    def predict(self):
        bvalid = 0
        rvalid = 0
        rdata = 0
        if self.write_counter:
            if self.write_counter == 1:
                bvalid = 1
        if self.read_counter:
            if self.read_counter == 1:
                rvalid = 1
        o = make_empty_axi4lite_s2m_dict()
        o['bvalid'] = bvalid
        o['rvalid'] = rvalid
        o['rdata'] = rdata
        return {
            'o': o,
        }

    def process(self, inputs):
        if inputs['i']['wvalid']:
            assert(inputs['i']['awvalid'] == 1)
            self.handle_write()
        if inputs['i']['arvalid']:
            self.handle_read()
        bvalid = 0
        rvalid = 0
        rdata = 0
        if self.write_counter:
            if self.write_counter == 1:
                bvalid = 1
            self.write_counter -= 1
        if self.read_counter:
            if self.read_counter == 1:
                rvalid = 1
            self.read_counter -= 1
        o = make_empty_axi4lite_s2m_dict()
        o['bvalid'] = bvalid
        o['rvalid'] = rvalid
        o['rdata'] = rdata
        if inputs['reset']:
            self.reset()
        return {
            'o': o,
        }


import datetime
from django.contrib.auth.decorators import login_required
from django.http import JsonResponse, HttpResponse
from django.shortcuts import render, redirect
from .forms import NewContactForm, NewCompanyForm
from .models import Company, Contact
from django.contrib.auth.models import User

@login_required(login_url='/')
def contacts(request):
    ''' get contact page  view '''
    contacts = Contact.objects.filter(user__username=request.user.username)
    context = {
        "contacts": contacts
    }
    return render(request, 'contacts/contacts.html', context)

@login_required(login_url='/')
def clients(request):
    ''' get clients page views '''
    clients = Company.objects.filter(
        user__username=request.user.username)
    context = {
        "clients": clients
    }
    return render(request, 'contacts/clients.html', context)

@login_required(login_url='/')
def clients_true(request):
    ''' get customers page views '''
    clients = Company.objects.filter(
        user__username=request.user.username,
        client=True)
    print(clients)
    context = {
        "clients": clients
    }
    return render(request, 'contacts/clients.html', context)

@login_required(login_url='/')
def prospects(request):
    ''' get prospects page views '''
    clients = Company.objects.filter(
        user__username=request.user.username,
        client=False)
    context = {
        "clients": clients
    }
    return render(request, 'contacts/clients.html', context)

@login_required(login_url='/')
def add_client(request):
    ''' add client '''
    if request.method == 'POST':
        form = NewCompanyForm(request.POST)
        if form.is_valid():
            try:
                # check the name and the city of company if it exists
                Company.objects.get(
                    cp_name=form.cleaned_data["cp_name"],
                    company_city=form.cleaned_data["company_city"],
                    user__username=request.user.username)
            except Company.DoesNotExist:
                # if company does not exists we create it
                form.save()
                company = Company.objects.get(
                    cp_name=form.cleaned_data["cp_name"],
                    company_city=form.cleaned_data["company_city"],
                    )
                company.user = request.user
                company.save()
                context = {
                    "company": "added"
                }
                return render(request, "contacts/thanks.html", context)
            else:
                # if client exists we go to next form
                context = {
                    "company": "exists"
                }
                return render(request,
                              "contacts/already_exists.html",
                              context)
    else:
        form = NewCompanyForm()
    context = {
        "form": form
    }
    return render(request, 'contacts/add_client.html', context)

@login_required(login_url='/')
def add_contact(request, company_id):
    ''' add contact view '''
    if request.method == 'POST':
        form = NewContactForm(request.POST, request.FILES)
        if form.is_valid():
            # check if contact exists
            try:
                Contact.objects.get(
                    name=form.cleaned_data["name"],
                    user__username=request.user.username,
                    email=form.cleaned_data["email"])
            # if contact does not exists we create it
            except Contact.DoesNotExist:
                form.save()
                context = {
                    "contact": "added"
                }
                return render(request, 'contacts/thanks.html',
                              context)
            else:
                # if contact exists we redirect to this page
                context = {
                    "contact": "exists"
                }
                return render(request,
                              "contacts/already_exists.html",
                              context)
    else:
        form = NewContactForm(username=request.user.username,
                              company_id=company_id)
    context = {
        "form": form,
    }
    return render(request, 'contacts/add_contact.html', context)

@login_required(login_url='/')
def client_details(request, company_id):
    ''' contact detail page '''
    company = Company.objects.get(id=company_id)
    contacts = Contact.objects.filter(
        user__username=request.user.username,
        company__id=company_id)
    context = {
        "company": company,
        "contacts": contacts
    }
    return render(request, "contacts/client_detail.html", context)

@login_required(login_url='/')
def Set(request, company_id):
    ''' set prospect to client view '''
    company = Company.objects.get(id=company_id)
    if company.client is True:
        company.client = False
    elif company.client is False:
        company.client = True
    company.save()
    data = {
        "client": company.client
    }
    return JsonResponse(data)

@login_required(login_url='/')
def client_prospects_percent(request):
    ''' client prospect percent '''
    total_companies = Company.objects.filter(
        user__username=request.user.username).count()
    total_clients = Company.objects.filter(
        client=True, user__username=request.user.username).count()
    total_prospects = Company.objects.filter(
        client=False, user__username=request.user.username).count()
    try:
        prospects_percent = (total_prospects / total_companies) * 100
    except ZeroDivisionError:
        prospects_percent = 0
    try:
        client_percent = (total_clients / total_companies) * 100
    except ZeroDivisionError:
        client_percent = 0
    data = {
        "total_companies": total_companies,
        "total_clients": total_clients,
        "total_prospects": total_prospects,
        "clients_percent": client_percent,
        "prospects_percent": prospects_percent
    }
    return JsonResponse(data)

@login_required(login_url='/')
def contact_month(request):
    ''' contacts per month '''
    year = datetime.date.today().year
    contact_counts_month = []
    username = request.user.username
    months = ['0' + str(n) if n < 10 else str(n) for n in range(1, 13)]
    for month in months:
        date = str(year) + '-' + month
        contact_count = Contact.objects.filter(
            date__startswith=date,
            user__username=request.user.username).count()
        contact_counts_month.append(contact_count)
    data = {
        "contact": contact_counts_month,
    }
    return JsonResponse(data)

@login_required(login_url='/')
def thanks(request):
    ''' thanks page view '''
    return render(request, 'contacts/thanks.html')

def p_404(request, exception):
    ''' 404 view '''
    return render(request, "404.html", status=404)

def p_500(request):
    ''' 500 view '''
    return render(request, "500.html", status=500)

@login_required(login_url='/')
def edit_comp(request, company_id):
    company = Company.objects.get(id=company_id)
    if request.method == "POST":
        form = NewCompanyForm(request.POST)
        if form.is_valid():
            company.cp_name = form.cleaned_data["cp_name"]
            company.address = form.cleaned_data["address"]
            company.company_country = form.cleaned_data["company_country"]
            company.company_city = form.cleaned_data["company_city"]
            company.field = form.cleaned_data["field"]
            company.description = form.cleaned_data["description"]
            company.email = form.cleaned_data["email"]
            company.phone_number = form.cleaned_data["phone_number"]
            company.user = request.user
            if form.cleaned_data["client"] == 'true':
                company.client = True
            else:
                company.client = False
            company.save()
            context = {
                "company": "edited"
            }
            return render(request, "contacts/edition_done.html",
                          context)
    else:
        data = {
            "cp_name": company.cp_name,
            "address": company.address,
            "company_country": company.company_country,
            "company_city": company.company_city,
            "field": company.field,
            "description": company.description,
            "client": company.client,
            "email": company.email,
            "phone_number": company.phone_number
        }
        form = NewCompanyForm(data)
    context = {
        "form": form
    }
    return render(request, "contacts/edit_client.html", context)

@login_required(login_url='/')
def edit_contact(request, contact_id):
    contact = Contact.objects.get(id=contact_id)
    request.session["c_id"] = contact.id
    if request.method == "POST":
        form = NewContactForm(request.POST, request.FILES)
        if form.is_valid():
            contact.name = form.cleaned_data["name"]
            contact.country = form.cleaned_data["country"]
            contact.city = form.cleaned_data["city"]
            contact.email = form.cleaned_data["email"]
            contact.phone_number = form.cleaned_data["phone_number"]
            contact.age = form.cleaned_data["age"]
            contact.dep_name = form.cleaned_data["dep_name"]
            contact.function = form.cleaned_data["function"]
            if "photo" in form.cleaned_data:
                contact.photo = form.cleaned_data["photo"]
            contact.user = request.user
            contact.save()
            context = {
                "contact": 'edited'
            }
            return render(request,
                          "contacts/edition_done.html",
                          context)
    else:
        data = {
            "name": contact.name,
            "country": contact.country,
            "city": contact.city,
            "email": contact.email,
            "phone_number": contact.phone_number,
            "age": contact.age,
            "dep_name": contact.dep_name,
            "function": contact.function,
            "photo": contact.photo.url
        }
        form = NewContactForm(data,
                              username=request.user.username,
                              company_id=contact.company.id)
    context = {
        "form": form
    }
    return render(request, "contacts/edit_contact.html", context)

@login_required(login_url='/')
def delete_contact(request, contact_id):
    contact = Contact.objects.get(id=contact_id)
    contact.delete()
    context = {
        "contact": "deleted"
    }
    return render(request, "contacts/deletion_done.html",
                  context)

@login_required(login_url='/')
def delete_company(request, company_id):
    company = Company.objects.get(id=company_id)
    company.delete()
    context = {
        "company": "deleted"
    }
    return render(request, "contacts/deletion_done.html",
                  context)


import cv2
import os

def getGabor(img, filters):
  res = cv2.filter2D(img, cv2.CV_8UC4, filters) # 2D滤波函数 kern为滤波模板
  return res

def getMatchNum(matches, ratio):
  '''返回特征点匹配数量和匹配掩码'''
  matchesMask = [[0, 0] for i in range(len(matches))]
  matchNum = 0
  for i, (m, n) in enumerate(matches):
    if m.distance < ratio * n.distance:  # 将距离比率小于ratio的匹配点筛选出来
      matchesMask[i] = [1, 0]
      matchNum += 1
  return (matchNum, matchesMask)

filters = cv2.getGaborKernel((11, 11), 11, 2.670353755551324, 11, 1.94, -1.5707963267948966)

# 读取文件
imgfiles = []
for root, dirs, files in os.walk("./img"):
  imgfiles = files

# 创建sift及flann初始化
# 创建SIFT特征提取器
sift = cv2.xfeatures2d.SIFT_create()
# 创建FLANN匹配对象
FLANN_INDEX_KDTREE = 0
indexParams = dict(algorithm=FLANN_INDEX_KDTREE, trees=5)
searchParams = dict(checks=50)
flann = cv2.FlannBasedMatcher(indexParams, searchParams)

# 遍历并缓存处理
cache = {}
for file in imgfiles:
  image = cv2.imread('./img/' + file, cv2.IMREAD_GRAYSCALE)
  cache[file] = {}
  cache[file]["kp"], cache[file]["des"] = sift.detectAndCompute(getGabor(image, filters), None)

# 遍历
for file1 in imgfiles:
  split_file_name1 = file1.split("_")
  img1kp = cache[file1]["kp"]
  img1des = cache[file1]["des"]

  for file2 in imgfiles:
    split_file_name2 = file2.split("_")
    kp2 = cache[file2]["kp"]
    des2 = cache[file2]["des"]
    matches = flann.knnMatch(img1des, des2, k=2)  # 匹配特征点，为了删除匹配点，指定k=2，对样本图每个特征点，返回两个匹配
    (matchNum, matchesMask) = getMatchNum(matches, 0.8)  # 通过比率条件，计算匹配度
    matchRatio = matchNum * 100 / len(matches)
    print(str(split_file_name1[0] == split_file_name2[0]) + "," + str(matchRatio) + "," + split_file_name1[0] + "," + split_file_name2[0])


import pytest
from .pages.main_page import MainPage
from .pages.login_page import LoginPage
from .pages.basket_page import BasketPage

from selenium import webdriver

@pytest.fixture(scope="function")
def browser():
    print("\nstart browser for test..")
    browser = webdriver.Chrome()
    yield browser
    print("\nquit browser..")
    browser.quit()

@pytest.mark.login_guest
class TestLoginFromMainPage():

    def test_guest_can_go_to_login_page(self, browser):
        link = "http://selenium1py.pythonanywhere.com/"
        page = MainPage(browser, link)
        page.open()
        page.go_to_login_page()
        login_page = LoginPage(browser, browser.current_url)
        login_page.should_be_login_page()

    def test_guest_should_see_login_link(self, browser):
        link = "http://selenium1py.pythonanywhere.com/"
        page = MainPage(browser, link)
        page.open()
        page.should_be_login_link()

def test_guest_cant_see_product_in_basket_opened_from_main_page(browser):
    link = "http://selenium1py.pythonanywhere.com/"
    page = BasketPage(browser, link)
    page.open()
    page.go_to_basket_page()
    page.should_be_basket_empty()


#-*- coding:utf-8 -*-
import os
import time
import json
import warnings
import data_helpers
import numpy as np
import tensorflow as tf
from text_cnn import TextCNN
from sklearn.externals import joblib
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report

warnings.filterwarnings("ignore")

# Parameters
# ==================================================

# Data loading parameters
tf.flags.DEFINE_string('data_file', './data/traindata', "Data source for the text data")
tf.flags.DEFINE_float('test_size', 0.05, "Percentage of data to use for validation and test (default: 0.05)")
tf.flags.DEFINE_integer('vocab_size', 9000, "Select words to build vocabulary, according to term frequency (default: 9000)")
tf.flags.DEFINE_integer('sequence_length', 500, "Padding sentences to same length, cut off when necessary (default: 500)")

# Model hyperparameters
tf.flags.DEFINE_integer('embedding_size', 128, "Dimension of word embedding (default: 128)")
tf.flags.DEFINE_string('filter_sizes', '3,4,5', "Filter sizes to use in convolution layer, comma-separated (default: '3,4,5')")
tf.flags.DEFINE_integer('num_filters', 128, "Number of filters per filter size (default: 128)")
tf.flags.DEFINE_float('dropout_keep_prob', 0.5, "Dropout keep probability (default: 0.5)")
tf.flags.DEFINE_float('l2_reg_lambda', 0.0, "L2 regularization lambda (default: 0.0)")
tf.flags.DEFINE_float('learning_rate', 0.001, "Learning rate for model training (default: 0.001)")
tf.flags.DEFINE_float('grad_clip', 3.0, "Gradients clipping threshold (default: 3.0)")

# Training parameters
tf.flags.DEFINE_integer('batch_size', 128, "Batch size (default: 128)")
tf.flags.DEFINE_integer("num_epochs", 20, "Number of training epochs (default: 20)")
tf.flags.DEFINE_string('init_embedding_path', None, "Using pre-trained word embedding, npy file format")
tf.flags.DEFINE_string('init_model_path', None, "Continue training from saved model at this path")
tf.flags.DEFINE_integer('evaluate_every', 50, "Evaluate model on val set after this many steps (default: 50)")

# Tensorflow parameters
tf.flags.DEFINE_boolean('allow_soft_placement', True, "Allow device soft device placement (default: True)")
tf.flags.DEFINE_boolean('log_device_placement', False, "Log placement of ops on devices (default: False)")
tf.flags.DEFINE_boolean('gpu_allow_growth', True, "GPU memory allocation mode (default: True)")

FLAGS = tf.flags.FLAGS
FLAGS._parse_flags()
print("\nParameters:")
for param, value in sorted(FLAGS.__flags.items()):
    print("{} = {}".format(param.upper(), value))
print("")

def train_cnn():
    # Data Preparation
    # ==================================================

    if FLAGS.init_embedding_path is not None:
        embedding = np.load(FLAGS.init_embedding_path)
        print("Using pre-trained word embedding which shape is {}\n".format(embedding.shape))
        FLAGS.vocab_size = embedding.shape[0]
        FLAGS.embedding_size = embedding.shape[1]

    if FLAGS.init_model_path is not None:
        assert os.path.isdir(FLAGS.init_model_path), "init_model_path must be a directory\n"
        ckpt = tf.train.get_checkpoint_state(FLAGS.init_model_path)
        assert ckpt, "No checkpoint found in {}\n".format(FLAGS.init_model_path)
        assert ckpt.model_checkpoint_path, "No model_checkpoint_path found in checkpoint\n"

    # Create root directory
    timestamp = str(int(time.time()))
    root_dir = os.path.join(os.path.curdir, 'runs', 'textcnn', 'trained_result_' + timestamp)
    os.makedirs(root_dir)

    # Load data
    print("Loading data...\n")
    x, y = data_helpers.load_data(FLAGS.data_file, FLAGS.sequence_length, FLAGS.vocab_size, root_dir=root_dir)
    FLAGS.num_classes = len(y[0])

    # Split dataset
    x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=FLAGS.test_size, stratify=y, random_state=0)
    x_val, x_test, y_val, y_test = train_test_split(x_test, y_test, test_size=0.5, random_state=0)

    # Training
    # ==================================================
    with tf.Graph().as_default():
        tf_config = tf.ConfigProto(
            allow_soft_placement=FLAGS.allow_soft_placement,
            log_device_placement=FLAGS.log_device_placement)
        tf_config.gpu_options.allow_growth = FLAGS.gpu_allow_growth

        with tf.Session(config=tf_config).as_default() as sess:
            cnn = TextCNN(
		vocab_size=FLAGS.vocab_size,
		embedding_size=FLAGS.embedding_size,
		sequence_length=FLAGS.sequence_length,
		filter_sizes=list(map(int, FLAGS.filter_sizes.split(","))),
		num_filters=FLAGS.num_filters,
            	num_classes=FLAGS.num_classes,
		learning_rate=FLAGS.learning_rate,
		grad_clip=FLAGS.grad_clip,
            	l2_reg_lambda=FLAGS.l2_reg_lambda)

            # Output directory for models and summaries
            out_dir = os.path.abspath(root_dir)
            print("Writing to {}...\n".format(out_dir))

            # Summaries for loss and accuracy
            tf.summary.scalar("loss", cnn.loss)
            tf.summary.scalar("accuracy", cnn.accuracy)
            merged_summary = tf.summary.merge_all()

            # Summaries dictionary
            train_summary_dir = os.path.join(out_dir, 'summaries', 'train')
            val_summary_dir = os.path.join(out_dir, 'summaries', 'val')
            train_summary_writer = tf.summary.FileWriter(train_summary_dir, sess.graph)
            val_summary_writer = tf.summary.FileWriter(val_summary_dir, sess.graph)

            # Checkpoint directory, will not create itself
            checkpoint_dir = os.path.abspath(os.path.join(out_dir, 'checkpoints'))
            checkpoint_prefix = os.path.join(checkpoint_dir, 'model.ckpt')
            if not os.path.exists(checkpoint_dir):
                os.makedirs(checkpoint_dir)
            saver = tf.train.Saver(tf.global_variables(), max_to_keep=1)

            # Initialize all variables
            sess.run(tf.global_variables_initializer())

            # Using pre-trained word embedding
            if FLAGS.init_embedding_path is not None:
                sess.run(cnn.embedding.assign(embedding))
                del embedding

            # Continue training from saved model
            if FLAGS.init_model_path is not None:
                saver.restore(sess, ckpt.model_checkpoint_path)

            # Training start
            print("Start training...\n")
            best_at_step = 0
            best_val_accuracy = 0
            for epoch in range(FLAGS.num_epochs):
                # Generate train batches
                train_batches = data_helpers.batch_iter(list(zip(x_train, y_train)), FLAGS.batch_size)
                start = time.time()
                for batch in train_batches:
                    # Training model on x_batch and y_batch
                    x_batch, y_batch = zip(*batch)
                    feed_dict = {cnn.input_x: x_batch, cnn.input_y: y_batch, cnn.keep_prob: FLAGS.dropout_keep_prob, cnn.is_training: True}
                    _, global_step, train_summaries, train_loss, train_accuracy = sess.run([cnn.train_op, cnn.global_step,
                        merged_summary, cnn.loss, cnn.accuracy], feed_dict=feed_dict)

                    # Evaluates model on val set
                    if global_step % FLAGS.evaluate_every == 0:
                        end = time.time()
                        train_summary_writer.add_summary(train_summaries, global_step)
                        feed_dict = {cnn.input_x: x_val, cnn.input_y: y_val, cnn.keep_prob: FLAGS.dropout_keep_prob, cnn.is_training: False}
                        val_summaries, val_loss, val_accuracy = sess.run([merged_summary, cnn.loss, cnn.accuracy], feed_dict=feed_dict)
                        val_summary_writer.add_summary(val_summaries, global_step)
                        print("Epoch: {}, global step: {}, training speed: {:.3f}sec/batch".format(epoch,
                            global_step, (end - start) / FLAGS.evaluate_every))
                        print("train loss: {:.3f}, train accuracy: {:.3f}, val loss: {:.3f}, val accuracy: {:.3f}\n".format(train_loss,
                            train_accuracy, val_loss, val_accuracy))
                        # If improved, save the model
                        if val_accuracy > best_val_accuracy:
                            print("Get a best val accuracy at step {}, model saving...\n".format(global_step))
                            saver.save(sess, checkpoint_prefix, global_step=global_step)
                            best_val_accuracy = val_accuracy
                            best_at_step = global_step
                        start = time.time()

            # Rename the checkpoint
            best_model_prefix = checkpoint_prefix + '-' + str(best_at_step)
            os.rename(best_model_prefix + '.index', os.path.join(checkpoint_dir, 'best_model.index'))
            os.rename(best_model_prefix + '.meta', os.path.join(checkpoint_dir, 'best_model.meta'))
            os.rename(best_model_prefix + '.data-00000-of-00001', os.path.join(checkpoint_dir, 'best_model.data-00000-of-00001'))

            # Testing on test set
            print("\nTraining complete, testing the best model on test set...\n")
            saver.restore(sess, os.path.join(checkpoint_dir, 'best_model'))
            feed_dict = {cnn.input_x: x_test, cnn.input_y: y_test, cnn.keep_prob: FLAGS.dropout_keep_prob, cnn.is_training: False}
            y_logits, test_accuracy = sess.run([cnn.logits, cnn.accuracy], feed_dict=feed_dict)
            print("Testing Accuracy: {:.3f}\n".format(test_accuracy))
            label_transformer = joblib.load(os.path.join(out_dir, 'label_transformer.pkl'))
            y_test_original = label_transformer.inverse_transform(y_test)
            y_logits_original = label_transformer.inverse_transform(y_logits)
            print("Precision, Recall and F1-Score:\n\n", classification_report(y_test_original, y_logits_original))

            # Save parameters
            print("Parameters saving...\n")
            params = {}
            for param, value in FLAGS.__flags.items():
                params[param] = value
            with open(os.path.join(out_dir, 'parameters.json'), 'w') as outfile:
                json.dump(params, outfile, indent=4, sort_keys=True, ensure_ascii=False)

            # Save word embedding
            print("Word embedding saving...\n")
            np.save(os.path.join(out_dir, 'embedding.npy'), sess.run(cnn.embedding))

if __name__ == '__main__':
    train_cnn()