import os
import time
import json
import requests

path = ""
# ex. "public/"

whitelist_file_path = "/opt/SentinelNetGuard/whitelist.txt"
cfg_file_path = "/opt/SentinelNetGuard/config.json"

class local_server:
    def __init__(self, pre_whitelist=None, unid="null", owner="null", lifetime=3600, destruct_time=-1):
        # Destruction time -1 == infinite alive_t
        if pre_whitelist is None:
            pre_whitelist = []
        self.unid = unid
        self.owner = owner
        self.lifetime = lifetime
        self.destruct_time = destruct_time
        self.time_created = ""
        self.connected_users = ""
        self.location = ""
        self.node_admins = ""
        self.server_url = "localhost"
        self.active = False
        self.whitelist_enabled = True
        self.pre_whitelist = pre_whitelist

        if self.lifetime == 0:
            self.lifetime = 3600

        # Get whitelist
        self.whitelist = self.get_whitelist()

        if not os.path.exists(whitelist_file_path):
            with open(whitelist_file_path, "w") as f:
                f.write("")

        # Add pre-whitelist to whitelist
        if self.pre_whitelist:
            for user in self.pre_whitelist:
                self.add_to_whitelist(user)

    def get_whitelist_enabled(self):
        return self.whitelist_enabled

    def whitelist_enabled_in_cfg(self):
        if get_config_key("whitelist") == "true":
            self.whitelist_enabled = True
        else:
            self.whitelist_enabled = False

    def get_whitelist(self):
        # if whitelist file doesn't exist, create it
        if not os.path.exists(whitelist_file_path):
            with open(whitelist_file_path, "w") as f:
                f.write("")
        # read whitelist from file
        with open(whitelist_file_path, "r") as f:
            whitelist = f.readlines()
        # remove whitespace characters like `\n` at the end of each line
        whitelist = [x.strip() for x in whitelist]
        return whitelist

    def add_to_whitelist(self, username):
        # add username to whitelist
        with open(whitelist_file_path, "a") as f:
            f.write(username + "\n")

    def remove_from_whitelist(self, username):
        # remove username from whitelist
        with open(whitelist_file_path, "r") as f:
            lines = f.readlines()
        with open(whitelist_file_path, "w") as f:
            for line in lines:
                if line.strip("\n") != username:
                    f.write(line)

    def set_active(self):
        """
        Set server to active in db.
        """
        # Get private key and unid from cfg
        priv_key = get_config_key("private_key")
        callback_type = "mgsengfse789fj39p2qjf8920qjf02q9d2a"
        unid = get_config_key("server_unid")
        url_actual = get_config_key("server_url")

        r = requests.get(
            f"https://{url_actual}/{path}create.php?auth={callback_type}&srv_host_callback=true&unid={unid}&pkey={priv_key}")
        # get echoed response
        response = r.text

        # Return node info
        if response == "true":
            self.active = True
            return True
        else:
            self.active = False
            return False

    def get_predefined_whitelist(self):
        """
        Get Predefined whitelist directly from DB.
        """
        # Get private key and unid from cfg
        priv_key = get_config_key("private_key")
        callback_type = "dfdwanh2a9ofdha2ohdfusiaolnfdgwoai"
        unid = get_config_key("server_unid")
        url_actual = get_config_key("server_url")

        r = requests.get(
            f"https://{url_actual}/{path}create.php?auth={callback_type}&srv_host_callback=true&unid={unid}&pkey={priv_key}")
        # get echoed response
        response = r.text

        if response == "false":
            return False
        else:
            return response

    def dec_lifetime(self):
        """
        Decrement lifetime by 1.
        """
        cur_lifetime = self.lifetime
        new_lifetime = cur_lifetime - 1
        self.lifetime = new_lifetime
        write_to_config_key("server_lifetime", str(new_lifetime))

    def shorten_node_lifetime(self, time_to_shorten: int):
        cur_lifetime = self.lifetime
        new_lifetime = cur_lifetime - time_to_shorten
        self.lifetime = new_lifetime
        write_to_config_key("server_lifetime", str(new_lifetime))

    def extend_node_lifetime(self, time_to_extend: int):
        # Hours to seconds
        time_to_extend = time_to_extend * 3600
        cur_lifetime = self.lifetime
        new_lifetime = cur_lifetime + time_to_extend
        self.lifetime = new_lifetime
        write_to_config_key("server_lifetime", str(new_lifetime))

    def get_lifetime(self):
        return self.lifetime

    def destruct(self):
        """
        Destruct.
        """
        priv_key = get_config_key("private_key")
        callback_type = "udap982unmdc98p2anucd28a9phjhxazjldh7a2"
        unid = get_config_key("server_unid")
        url_actual = get_config_key("server_url")

        r = requests.get(
            f"https://{url_actual}/{path}destruction_manager.php?auth={callback_type}&destructive_action_actual=true&unid={unid}&pkey={priv_key}")
        # get echoed response
        response = r.text
        if response == "true":
            return True
        else:
            return False

    def delete_files(self):
        """
        Delete files.
        """
        os.system("rm -rf /opt/SentinelNetGuard")
        return True

    def set_inactive(self):
        """
        Set server to active in db.
        """
        # Get private key and unid from cfg
        priv_key = get_config_key("private_key")
        callback_type = "dha278hda280hd7829a9cdn7897892"
        unid = get_config_key("server_unid")
        url_actual = get_config_key("server_url")

        r = requests.get(
            f"https://{url_actual}/{path}create.php?auth={callback_type}&srv_host_callback=true&unid={unid}&pkey={priv_key}")
        # get echoed response
        response = r.text

        # Return node info
        if response == "true":
            self.active = False
            return True
        else:
            self.active = True
            return False

    def update_global_host_info(self):
        """
        Get ip and port from cfg and contact webserver
        """
        specified_ip_pub = get_config_key("public_ip")
        specified_ip_internal = get_config_key("host_ip")
        specified_port = get_config_key("host_port")

        url_actual = get_config_key("server_url")
        callback_type = "vdnaa2ood2ha7dh2ahajklhcxjzndghoan"
        unid = get_config_key("server_unid")
        priv_key = get_config_key("private_key")

        whitelist = self.get_whitelist()
        whitelist = ",".join(whitelist)
        if whitelist == "":
            whitelist = "null"

        successful = requests.get(
            f"https://{url_actual}/{path}create.php?auth={callback_type}&srv_host_callback=true&global_update=true&unid={unid}&priv_key={priv_key}&ip_pub={specified_ip_pub}&ip_internal={specified_ip_internal}&port={specified_port}&whitelist={whitelist}")
        # Convert response to json
        # print(successful)
        # print(successful.text)
        # Return node info
        if successful.text == "true":
            return True
        else:
            return False

    def get_node_info(self):
        """
        Get node information from webserver callback.
        """

        # Get private key and unid from cfg
        priv_key = get_config_key("private_key")
        callback_type = "ghwhs907fy3vj7890fmw7vh387r3h8f7h3d"
        unid = get_config_key("server_unid")
        url_actual = get_config_key("server_url")

        r = requests.get(
            f"https://{url_actual}/{path}create.php?auth={callback_type}&srv_host_callback=true&unid={unid}&pkey={priv_key}")
        # Convert response to json
        node_info = r.json()
        del r
        # Return node info
        return node_info

    def fetch_node_info(self):
        try:
            info = self.get_node_info()
            # Set self variables to node info
            self.unid = info["unid"]
            self.owner = info["owned_by"]
            self.time_created = info["time_created"]
            self.connected_users = info["connected_users"]
            self.location = info["location"]
            self.node_admins = info["node_admins"]

            # Update local config
            self.update_local_cfg()

            return True

        except Exception as e:
            print(f"ERROR {e}")
            return False

    def validate_destruction(self):
        """
        Validate if destruction time has passed.
        :return:
        true or false
        """
        if self.destruct_time == -1:
            return False
        else:
            if self.destruct_time < time.time():
                return True
            else:
                return False

    def update_local_cfg(self):
        try:
            write_to_config_key("server_unid", str(self.unid))
            write_to_config_key("server_owner", str(self.owner))
            write_to_config_key("server_lifetime", str(self.lifetime))
            write_to_config_key("server_destruct_time", str(self.destruct_time))
            write_to_config_key("time_created", str(self.time_created))
            write_to_config_key("connected_users", str(self.connected_users))
            write_to_config_key("location", str(self.location))
            write_to_config_key("node_admins", str(self.node_admins))
        except Exception as e:
            print(e)

def write_to_config_key(key: str, value: str):
    """
    Writes to json config file
    """
    try:
        with open(cfg_file_path, 'r') as json_file:
            json_decoded = json.load(json_file)

        json_decoded[key] = value

        with open(cfg_file_path, 'w') as json_file:
            json.dump(json_decoded, json_file, indent=4)
    except Exception as e:
        print(e)

def get_config_key(specific_key: str):
    """
    Reads from json config file key
    """
    with open(cfg_file_path) as json_file:
        json_decoded = json.load(json_file)

    return json_decoded[f'{specific_key}']


import numpy as np
import cv2
import matplotlib.pyplot as plt
import torch
import AENetwork
from cfg import DefaultConfig
# import time

torch.backends.cudnn.deterministic = True  # 因为使用了GPU，需要保证种子固定才能保证每次结果相同
torch.backends.cudnn.benchmark = True  # 让cuDNN找到最合适的算法
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(device)

model = AENetwork.AENetwork()
model.eval()  # 不使用BN和dropout，因为测试batch太小，会被BN和dropout影响
model.load_state_dict(torch.load(DefaultConfig.load_model_path + 'network_model.pth'))  # 读模型
model.to(device)

# 读取图像进行预处理，转换为tensor
image_path = r'./data/bottle//test/defect/295.jpg'
img = cv2.imread(image_path)
image = cv2.resize(img, (256, 256))
image_resized = (image.astype('float32') / 127.5) - 1.0  # to -1 ~ 1
image_resized = np.transpose(image_resized, [2, 0, 1])
input_tensor = torch.from_numpy(image_resized).to(device).unsqueeze(0)

output_image = model(input_tensor)
# 需要优化
# output_image = output_image.clone().squeeze(0).cpu().detach().numpy()
output_image = output_image.clone().squeeze(0).detach().cpu().numpy()
cv2_frame = ((output_image + 1) * 127.5).transpose(1, 2, 0).astype('uint8')

# 图像求差得到缺陷图
res_image = cv2.cvtColor(np.power(cv2.subtract(cv2_frame, image), 1), cv2.COLOR_BGR2GRAY)
res_image = cv2.medianBlur(res_image, 3)

# 打印结果
# plt.rcParams['font.sans-serif'] = ['SimHei']
# plt.rcParams['axes.unicode_minus'] = False  # 输出中文字体
# plt.figure(figsize=(19, 19))
# plt.subplot(1, 3, 1)
# plt.imshow(image)
# plt.title('原始图像')
# plt.subplot(1, 3, 2)
# plt.imshow(cv2_frame)
# plt.title('重构图像')
# plt.subplot(1, 3, 3)
# plt.imshow(res_image)
# plt.title('残差图像')
# plt.show()

# cost 0.030434608459472656 s
# cost 0.1248326301574707 s
# cost 0.007006168365478516 s
# cost 2.6321322917938232 s
# cost 0.0009963512420654297 s

plt.rcParams['font.sans-serif'] = ['SimHei']
plt.rcParams['axes.unicode_minus'] = False  # 输出中文字体
plt.figure(figsize=(19, 19))
plt.subplot(2, 1, 1)
plt.imshow(image)
plt.title('原始图像')
plt.subplot(2, 1, 2)
plt.imshow(res_image)
plt.title('残差图像')
plt.show()


import tensorflow as tf
import numpy as np
import os
import cifar10_input as c
from PIL import Image

cur_dir = os.getcwd()
print("resizing images")
print("current directory:",cur_dir)

def modify_image(image):
    flipped_images = tf.image.flip_up_down(image)
    return flipped_images

def read_image(filename_queue):
    reader = tf.WholeFileReader()
    key,value = reader.read(filename_queue)
    image = tf.image.decode_jpeg(value)
    return image

def inputs():
    filenames = ['/tmp/isic_dataset/bin_32_0.bin']
    filename_queue = tf.train.string_input_producer(filenames)
    read_input = c.read_cifar10(filename_queue)
    return modify_image(read_input.uint8image)

with tf.Graph().as_default():
    image = inputs()
    init = tf.initialize_all_variables()
    sess = tf.Session()
    sess.run(init)
    tf.train.start_queue_runners(sess=sess)
    for i in range(10):
        img = sess.run(image)
        img = Image.fromarray(img, "RGB")
        img.save(os.path.join(cur_dir,"foo"+str(i)+".jpeg"))


import ast
from functools import partial

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import scipy.stats
import seaborn as sn
import torch
from tqdm import tqdm

pd.set_option("mode.chained_assignment", None)
SCALE = 13
HEIGHT_SCALE = 0.5
sn.set(rc={"figure.figsize": (SCALE, int(HEIGHT_SCALE * SCALE))})
sn.set(font_scale=2.0)
sn.set_style(style="white")
sn.color_palette("colorblind")

LEGEND_Y_CORD = -0.75  # * (HEIGHT_SCALE / 2.0)
SUBPLOT_ADJUST = 1 / HEIGHT_SCALE  # -(0.05 + LEGEND_Y_CORD)
LEGEND_X_CORD = 0.45

plt.gcf().subplots_adjust(bottom=0.40, left=0.2, top=0.95)

PLOT_FROM_CACHE = False
PLOT_SAFTEY_MARGIN = 1.25

N = 3  # Significant Figures for Results
DP = 5

np.random.seed(999)
torch.random.manual_seed(999)

def is_float(element) -> bool:
    try:
        float(element)
        return True
    except ValueError:
        return False

def string_to_float_dict(d):
    return {k: float(v) if is_float(v) else v for k, v in d.items()}

def ci(data, confidence=0.95):
    # https://stackoverflow.com/questions/15033511/compute-a-confidence-interval-from-sample-data
    a = 1.0 * np.array(data)
    n = len(a)
    m, se = np.mean(a), scipy.stats.sem(a)  # noqa: F841  # pylint: disable=unused-variable
    h = se * scipy.stats.t.ppf((1 + confidence) / 2.0, n - 1)
    return h

DEVICE = "cuda" if torch.cuda.is_available() else "cpu"

X_METRIC = "nevals"
Y_METRIC = "nmse_train"

def mean_confidence_interval(data, confidence=0.95):
    a = 1.0 * np.array(data)
    n = len(a)
    m, se = np.mean(a), scipy.stats.sem(a)
    h = se * scipy.stats.t.ppf((1 + confidence) / 2.0, n - 1)
    return m, 2 * h  # m-h, m+h

def confidence_interval(prob, n):
    return 1.96 * np.sqrt((prob * (1 - prob)) / n)

N = 5  # Significant figure

# LOG_PATH = "./process_results/results/all_run_recovery_multi-20220924-180429-fn_d_2_log_40_f.txt"
# LOG_PATH = "./process_results/results/all_run_recovery_multi-20220924-180429-l_cd_12_log_20.txt"
# LOG_PATH = "./process_results/results/all_run_recovery_multi-20220924-180429-l_cd_12_log_og_f.txt"
# LOG_PATH = "./process_results/results/all_run_recovery_multi-20220924-180429-fn_d_all_log_f.txt"
LOG_PATH = "./process_results/results/all_run_recovery_multi-20220924-180429-fn_d_5_log_existing.txt"

GENERATE_FIGS = False

if __name__ == "__main__":
    metric_to_plot = "acc_iid"
    with open(LOG_PATH) as f:
        lines = f.readlines()

    # datasets = {}
    pd_l = []
    df_tmp = []  # Drop last entry if not completed
    acc_iid = 0
    acc_ood = 0
    for line in tqdm(lines):
        if "[Test epoch=" in line:
            line = line.replace("eqs_invalid %", "eqs_invalid%")
            epoch_dict = {a.split("=")[0]: a.split("=")[1] for a in line.split("[Test")[1].strip().split() if a != "|"}
            epoch_dict["epoch"] = epoch_dict["epoch"][:-1]
            epoch_dict = string_to_float_dict(epoch_dict)
            r_best = epoch_dict["r_best"]
            epoch_dict["nmse_train"] = (1 / r_best - 1) / r_best  # pyright: ignore
            if r_best == 1.0:
                acc_iid = epoch_dict["acc_iid"]
                acc_ood = epoch_dict["acc_ood"]
            elif epoch_dict["epoch"] == 200.0:
                acc_iid = epoch_dict["acc_iid"]
                acc_ood = epoch_dict["acc_ood"]
        if "[TEST RESULT] {" in line:
            dl = [t for t in line.split("{")[1][:-2].split(", ") if "program" not in t]
            ddict = ast.literal_eval("{" + ", ".join(dl) + "}")
            dataset = ddict["dataset"]
            baseline = ddict["baseline"]
            success = ddict["success"]
            ddict["acc_iid"] = acc_iid
            ddict["acc_ood"] = acc_ood
            pd_l.append(ddict)

    dfm = pd.DataFrame(pd_l)

    if GENERATE_FIGS:
        dataset_plotting = ""
        for (dataset, baseline), group in dfm.groupby(["dataset", "baseline"]):
            if dataset_plotting == "":
                dataset_plotting = dataset
            elif dataset_plotting != dataset:
                plt.legend(
                    loc="lower center",
                    bbox_to_anchor=(LEGEND_X_CORD, LEGEND_Y_CORD),
                    ncol=1,
                    fancybox=True,
                    shadow=True,
                )
                plt.xlabel("#Evaluations")
                plt.ylabel("NMSE")
                plt.savefig(f"./results/{dataset_plotting}.png")
                plt.savefig(f"./results/{dataset_plotting}.pdf")
                plt.clf()
                dataset_plotting = dataset
            plt.plot(group[X_METRIC].to_numpy(), group[Y_METRIC].to_numpy(), label=baseline)
        plt.legend(
            loc="lower center", bbox_to_anchor=(LEGEND_X_CORD, LEGEND_Y_CORD), ncol=1, fancybox=True, shadow=True
        )
        plt.xlabel("#Evaluations")
        plt.ylabel("NMSE")
        plt.savefig(f"./results/{dataset_plotting}.png")
        plt.savefig(f"./results/{dataset_plotting}.pdf")
        plt.clf()

    n = dfm.run_seed.nunique()
    confidence_interval_data = partial(confidence_interval, n=n)

    assert not dfm.success.isnull().values.any(), "Nan values in the "  # pyright: ignore
    # .fillna(False)
    dfm["dataset"] = dfm.dataset.apply(lambda x: int(x.split("_")[-1]))

    df_out = dfm.groupby(["dataset", "baseline"]).agg([np.mean, ci]).reset_index()
    add_zero = False

    for baseline in df_out.baseline.unique():
        df_baseline = df_out[df_out.baseline == baseline]
        if add_zero:
            avg_recovery_rate = np.concatenate((df_baseline.success["mean"].to_numpy(), np.array([0]))).mean() * 100
            avg_recovery_rate_ci = (
                np.concatenate(
                    (df_baseline.success["mean"].map(confidence_interval_data).to_numpy(), np.array([0]))
                ).mean()
                * 100
            )
        else:
            avg_recovery_rate = df_baseline.success["mean"].mean() * 100
            avg_recovery_rate_ci = df_baseline.success["mean"].map(confidence_interval_data).mean() * 100
        print(f"{baseline}: {avg_recovery_rate:.2f} +/- {avg_recovery_rate_ci:.2f}")

    dfm_only_true = dfm[dfm.success == True]  # noqa: E712
    df_out = dfm.groupby(["dataset", "baseline"]).agg([np.mean, ci, np.std]).reset_index()

    average_equations_for_datasets_l = []
    for dataset in df_out.dataset.unique():
        df_dataset = df_out[df_out.dataset == dataset]
        if (df_dataset[df_dataset.baseline == "DSO"]["success"]["mean"] > 0).iloc[0] and (
            df_dataset[df_dataset.baseline == "DGSR-PRE-TRAINED"]["success"]["mean"] > 0
        ).iloc[0]:
            for baseline in df_out.baseline.unique():
                average_equations_for_datasets_l.append(
                    {
                        "dataset": dataset,
                        "baseline": baseline,
                        "n_samples_mean": df_dataset[df_dataset.baseline == baseline]["n_samples"]["mean"].iloc[0],
                        "n_samples_std": df_dataset[df_dataset.baseline == baseline]["n_samples"]["std"].iloc[0],
                        "n_samples_ci": df_dataset[df_dataset.baseline == baseline]["n_samples"]["ci"].iloc[0],
                    }
                )
    average_equations_for_datasets = pd.DataFrame(average_equations_for_datasets_l)
    for baseline in average_equations_for_datasets.baseline.unique():
        df_baseline = average_equations_for_datasets[average_equations_for_datasets.baseline == baseline]
        print(
            f"{baseline}: Avg. Eq. Evals. "
            f"{df_baseline.n_samples_mean.mean():,.2f} +/- {df_baseline.n_samples_ci.mean():,.2f}"
        )

    print("Final results now")

    a_out = dfm[["dataset", "baseline", "run_seed", "seed", "success", "n_samples"]]
    print(a_out.sort_values(by=["dataset", "baseline", "run_seed"]).reset_index().to_string())

    print(a_out.groupby(["dataset", "baseline"]).agg("mean").reset_index().to_string())
    print("")

    for index, row in df_out.iterrows():
        print(
            f"{row['dataset'].iloc[0]} {row['baseline'].iloc[0]} : {row.n_samples['mean']:,.0f} "
            # pylint: disable-next=anomalous-backslash-in-string
            f"$\pm$ {row.n_samples['ci']:,.0f}"  # noqa: W605  # pyright: ignore
        )

    print("")


import torch
from torch.autograd import Variable
from torch.utils.data import TensorDataset, DataLoader

import numpy as np

def to_Variables(*args, cuda=False):
    ret = []
    for arg in args:
        if cuda:
            ret.append(Variable(arg).cuda())
        else:
            ret.append(Variable(arg))

    return ret

def check_and_convert_tensor(x):
    if torch.is_tensor(x):
        return x.cpu().numpy()
    else:
        return x.data.cpu().numpy()

n_features = 11
n_samples = 250
n_classes = 2

# Should give us a random label vector with entries 1 and -1
test_labels = torch.ones(n_samples).long()
test_labels[torch.ones(test_labels.shape).uniform_() > 0.5] = -1

# Generate random test data
test_data = torch.rand(n_samples, n_features).double()

# s indices
s = [8]

# Convert the 8th column of x to be categorical
l = torch.ones(test_data[:, s[0]].shape).uniform_()
test_data[:, s[0]] = 2
test_data[:, s[0]][l < 0.67] = 1
test_data[:, s[0]][l < 0.33] = 0

# Conversions to variables for use inside of everything
x = Variable(test_data).double()
y = Variable(test_labels).long()

# Generate random weights
w = Variable(torch.rand(n_features, 1)).double()

# Convert the data into a torch dataset
ds = TensorDataset(test_data, test_labels)
dl = DataLoader(ds, batch_size=32, shuffle=True)

# Enumerate through the data
for i, data in enumerate(dl):
    inputs, labels = to_Variables(*data)
    dtype = type(inputs.data)

    # Build up the indices
    fairness_idxs = []
    vals = []
    divisors = []
    for col in s:
        vals.append(np.unique(check_and_convert_tensor(test_data)[:, col]))
        fairness_idxs.append([])
        prod = 1
        for val in vals[-1]:
            idxs = (test_data[:, col] == float(val)).nonzero().squeeze()
            prod *= len(idxs)
            fairness_idxs[-1].append(idxs)
        divisors.append(prod)

    # Actually calculate the penalty
    penalty = 0
    for idxs, val, div in zip(fairness_idxs, vals, divisors):
        # Should be size (n_classes test_data self.minibatch_size)
        local_dots = torch.mm(inputs, w)
        for idx in idxs:
            # Because of broadcasting rules, this should create a matrix of absolute differences
            # of size (self.minibatch_size test_data |S_i: class[idx]|)
            diff = torch.abs(labels.view(1, -1) - y[idx].view(-1, 1)).type(dtype)

            # Size: (n_classes test_data |S_i: class[idx]|)
            non_local_dot = torch.mm(x[idx], w)

            cross_term = 0
            for local_dot in local_dots.t():
                # Individual version
                cross_term = cross_term + (local_dot - non_local_dot) ** 2
                penalty = penalty + ((diff / div) * cross_term).sum()

                # # Group version
                # cross_term = cross_term + (local_dot - non_local_dot)
                # penalty = penalty + ((diff / div) * cross_term).sum() ** 2

    print(penalty)


import sys
from pathlib import Path
from typing import Any, Optional

import loguru
import loguru._logger

logger = loguru.logger

def setup_logger(
    output_dir: Optional[Path] = None,
    bind: dict[str, Any] = {},
    stderr_level: str = "INFO",
    file_level: str = "DEBUG",
    # ) -> "loguru.Logger":
) -> None:
    _logger = loguru.logger.bind(**bind)
    _logger.remove()
    format = (
        "<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</green> | "
        "<level>{level: <8}</level> | "
        "<cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan>"
        " - <level>{message}</level>"
        " - {extra}"
    )
    _logger.add(sys.stdout, colorize=True, format=format, level=stderr_level)
    if output_dir is not None:
        _logger.add(
            output_dir / "output.log",
            backtrace=True,
            diagnose=True,
            format=format,
        )
        _logger.add(
            output_dir / "output.jsonl", serialize=True, level=file_level
        )
        _logger.info(f"Use logger output_dir: {str(output_dir)}")

    global logger
    logger = _logger


from PIL import Image
from pytesseract import pytesseract, Output
import os
import spacy
import cv2
import streamlit as st

pytesseract.tesseract_cmd = r"C:/Program Files/Tesseract-OCR/tesseract.exe"

def img_to_txt(file_name):
    img = Image.open(file_name)
    print('image is loaded')
    text = pytesseract.image_to_string(file_name)
    return text

def gene_interest():
    file_name = os.path.basename(filename).split('.')[0]
    if '_' in file_name:
        underscore_index = file_name.index('_')
        file_name = file_name[:underscore_index]
    print(file_name)

def extract_keywords(text):
    gene_interest()
    line_list = []
    result_index = 0
    paragraphs = text.split('\n\n')
    for line in paragraphs:
        if ('result' or 'summary') in line.lower():
            result_index = paragraphs.index(line)
            break
    results = paragraphs[result_index:]
    results_paragraphs = '\n\n'.join(results)
    nlp = spacy.load("en_ner_bionlp13cg_md")
    if results_paragraphs != []:
        doc = nlp(results_paragraphs)
    else:
        doc = nlp(text)
    words = doc.ents
    keywords = list(words)
    keyword_strings = []
    for item in keywords:
        keyword_strings.append(str(item).rstrip())
    keyword_strings = list(dict.fromkeys(keyword_strings))
    return keyword_strings

def check_similarity(a, b):
    chars_to_remove = "1ltLT"
    for char in chars_to_remove:
        a = a.replace(char, "")
        b = b.replace(char, "")
    if (a in b):
        return True
    return False

def label_image(file_name):
    gene_name = os.path.basename(file_name).split(".")[0].split("_")[0]

    # Check if found
    found = False
    img = cv2.imread(file_name)
    d = pytesseract.image_to_data(img, output_type=Output.DICT)
    keys = list(d.keys())

    # Iterate
    n_boxes = len(d['text'])
    for i in range(n_boxes):
        if check_similarity(gene_name, d['text'][i]):
            found = True
            (x, y, w, h) = (d['left'][i], d['top'][i], d['width'][i], d['height'][i])
            img = cv2.rectangle(img, (x - 6, y - 6), (x + w + 9, y + h + 9), (0, 255, 0), 2)

    if (found == False):
        font = cv2.FONT_HERSHEY_SIMPLEX
        img = cv2.putText(img,'No gene was found on report',(100,500), font, 2, (255, 0, 0), 2, cv2.LINE_AA)

    cv2.imwrite('output.png', img) # Modify this
    return found

def file_selector(folder_path='.'):
    filenames = os.listdir(folder_path)
    selected_filename = st.selectbox("Select a file", filenames)
    return os.path.join(folder_path, selected_filename)

if __name__ == '__main__':
    if st.checkbox("select a file in current directory"):
        folder_path = '.'
        if st.checkbox('Change directory'):
            folder_path = st.text_input('Enter folder path')
        filename = file_selector(folder_path=folder_path)
        image = Image.open(filename)
        st.write('You selected `%s`' % filename)
        label = label_image(filename)
        if label is True:
            label = "your target gene is present in the report."
        else:
            label = "Your target gene is not present in the report."
        st.image('output.png', caption=label)
        keyword_text = "Here are the keywords that might be helpful:"
        st.text(keyword_text)
        st.text(extract_keywords(img_to_txt(filename)))


import pytest
import allure

@allure.feature('omplogin-分销商登录')
class TestDemo8_omplogin(object):

    @allure.story('omp登录有效类')
    @allure.title('1111111111111')
    @allure.severity('Normal')
    @allure.step('输入正确的用户名和密码')
    def test_loginYXL(self):
        '''
        用例描述：输入正确的用户名和密码可正常登录
        '''
        print('我是有效类1111')
        assert 1==1

    @allure.story('omp登录无效类')
    @allure.title('2222222222222')
    @allure.severity('Critical')
    @allure.issue('http://www.baidu.com',name='链接地址')
    @allure.testcase('http://obp.888ly.cn',name='bug地址')
    @allure.step('输入错误的用户名和密码组合')
    def test_loginWXL(self):
        '''
        用例描述：用户名+密码 各种错误或者为空的组合验证
        '''
        with allure.step('输入用户名'):
            print('我是无效类1111')
        with allure.step('输入密码'):
            print('输入密码')
        with allure.step('输入错误的用户名{0}，输入正确的密码{1}'.format(None,'abc123')):
            print('33333333333')
        with open(r'C:\Users\suqiangqiang\Desktop\33.png','rb') as f:
            file=f.read()
            allure.attach(file,'登录无效类--用户名为空',allure.attachment_type.PNG)
        assert 1==1

if __name__ == "__main__":
   pytest.main(['-s',__file__,'--alluredir','Learning/pytest/report/'])


#!/usr/bin/env python

from pprint import pprint
from napalm import get_network_driver
import my_devices

def task1b(dev):
    device_type = dev.pop("device_type")
    driver = get_network_driver(device_type)

    device = driver(**dev)
    device.open()
    print(device,"\n")
    facts = device.get_facts()
    pprint(facts)
    print('\n')
    print('The Platform Type is:',device_type)

if __name__ == "__main__":

    #task1b(my_devices.cisco3)

    for d in my_devices.all_devices:
        task1b(d)


from src.utils.testmodelutils import test_models_on_batch_and_show
from src.utils.Metrics import auc, acc, highest_tpr_thresh, lowest_fpr_thresh
from src.utils.ParamTuner import ParamTuner
from src.utils.utils import init_folder, make_folder_if_not_there
from src.utils.Plotting import make_plot

from src.experiments.utils import get_resize_wrapper, get_pretrained_size_and_norm_wrapper

import torch

import os

def RunExpt(expt_name,
            model,
            model_kwargs,
            epochs,
            directories,
            augmented,
            wrapper=None,
            lrs=[0.003],
            bss=[64],
            opts=[torch.optim.Adam],
            losses=[torch.nn.BCELoss],
            workers=2,
            save_every=2,
            **kwargs):
    """
    Run an experiment
    Basically just bunch all the things I need together for hyperparam tuning so I dont have to always write it out
    also now we can wrap it up for different models aswell
    :param expt_name: the name of the experiment
    :param model:the class of the model to use for tuning
    :param model_kwargs: the arguments to pass to the model initialise
    :param epochs: the number of epochs to train for
    :param directories: a dictionary of the data, log, and checkpoint directories
    :param augmented: whether this experiment is using augmented data or not
    :param wrapper: a wrapper function for data loading if needed
    :param lrs: a list of learning rates to try length >= 1
    :param bss: a list of batch sizes to try length >= 1
    :param opts: a list of optimizers to try length >= 1
    :param losses: a list of loss functions to try length >= 1
    :param workers: the number of workers for dataloading to use
    :param save_every: how often to save the models and optimizers
    :param kwargs: the optional kwargs to pass to the param tuner
    """
    torch.cuda.empty_cache()

    DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")

    print(f"Using Device {DEVICE}")

    EPOCHS = epochs
    WORKERS = workers

    metrics_dict = {
        "auc": auc,
        "acc": acc,
        "top_tpr_thresh": highest_tpr_thresh,
        "low_fpr_thresh": lowest_fpr_thresh
    }

    log_dir = init_folder(f"{expt_name}", directories["log"], True)
    ckp_dir = init_folder(f"{expt_name}", directories["ckp"], True)

    tune = ParamTuner(expt_name,
                      model,
                      metrics_dict,
                      "auc",
                      directories["data"],
                      log_dir,
                      ckp_dir,
                      augmented,
                      model_kwargs=model_kwargs,
                      learning_rates=lrs,
                      batch_sizes=bss,
                      optimisers=opts,
                      losses=losses,
                      EPOCHS=EPOCHS,
                      wrapped_function=wrapper,
                      WORKERS=WORKERS,
                      SAVE_EVERY=save_every,
                      DEVICE=DEVICE,
                      verbose=True,
                      overwrite=True,
                      **kwargs)

    print(f"HYPER-PARAM OUTPUT: {tune.trials}")
    print(f"EXPERIMENT {expt_name} FINISHED")

    print("PLOTTING!")

    cwd = os.getcwd()
    fig_dir = make_folder_if_not_there("figs", cwd)
    tuned_fig_dir = init_folder(expt_name, fig_dir, True)
    make_plot(log_dir, tuned_fig_dir)

    test_data_path = ""
    if "test" in directories.keys():
        test_data_path = directories["test"]
    else:
        test_data_path = directories["data"]
    test_models_on_batch_and_show(expt_name, test_data_path, ckp_dir, model, model_kwargs, dev=DEVICE, wrapped=wrapper)

    torch.cuda.empty_cache()

def base_experiment(name,
                    model,
                    directories,
                    workers,
                    model_kwargs,
                    aug,
                    wrapper=None,
                    lrs=[0.1, 0.01, 0.001, 0.0001],
                    bss=[32, 64],
                    opts=[torch.optim.Adam],
                    losses=[torch.nn.BCELoss],
                    **kwargs):
    """
    Run a single experiment
    :param name: name of experiment
    :param model: the class of the model to use
    :param directories: a dictionary of the data, logs, and checkpoints
    :param workers: the number of workers to use for data loading
    :param model_kwargs: the arguments to pass to the model when initialized
    :param aug: wether ot not the data is augmented
    :param wrapper: Wrapper required
    :param lrs: The learning rates to try
    :param bss: The Batch Sizes to try
    :param opts: The Optimizers to try
    :param losses: The Loss Functions to try
    :param kwargs: the optional kwargs to pass to the param tuner
    """
    RunExpt(f"{name}_EXPT",
            model,
            model_kwargs,
            100,
            directories,
            aug,
            wrapper,
            lrs=lrs,
            bss=bss,
            opts=opts,
            losses=losses,
            workers=workers,
            save_every=10,
            **kwargs)

def regular_experiment(name,
                       model,
                       directories,
                       size,
                       workers,
                       model_kwargs,
                       aug,
                       lrs=[0.1, 0.01, 0.001, 0.0001],
                       bss=[32, 64],
                       opts=[torch.optim.Adam],
                       losses=[torch.nn.BCELoss],
                       **kwargs):
    """
    Run a single experiment on an untrained model
    :param name: name of experiment
    :param model: the class of the model to use
    :param directories: a dictionary of the data, logs, and checkpoints
    :param size: size to resize samples to
    :param workers: the number of workers to use for data loading
    :param model_kwargs: the arguments to pass to the model when initialized
    :param aug: wether ot not the data is augmented
    :param lrs: The learning rates to try
    :param bss: The Batch Sizes to try
    :param opts: The Optimizers to try
    :param losses: The Loss Functions to try
    :param kwargs: the optional kwargs to pass to the param tuner
    """
    wrapper = get_resize_wrapper(size)
    base_experiment(name, model, directories, workers, model_kwargs, aug, wrapper, lrs=lrs, bss=bss, opts=opts,
                    losses=losses, **kwargs)

def pretrained_experiment(name,
                          model,
                          directories,
                          size,
                          workers,
                          model_kwargs,
                          aug,
                          lrs=[0.1, 0.01, 0.001, 0.0001],
                          bss=[32, 64],
                          opts=[torch.optim.Adam],
                          losses=[torch.nn.BCELoss],
                          **kwargs):
    """
    Run a single experiment on a pretrained model
    :param name: name of experiment
    :param model: the class of the model to use
    :param directories: a dictionary of the data, logs, and checkpoints
    :param size: size to resize samples to
    :param workers: the number of workers to use for data loading
    :param model_kwargs: the arguments to pass to the model when initialized
    :param aug: wether ot not the data is augmented
    :param lrs: The learning rates to try
    :param bss: The Batch Sizes to try
    :param opts: The Optimizers to try
    :param losses: The Loss Functions to try
    :param kwargs: the optional kwargs to pass to the param tuner
    """
    wrapper = get_pretrained_size_and_norm_wrapper(size)
    base_experiment(name, model, directories, workers, model_kwargs, aug, wrapper, lrs=lrs, bss=bss, opts=opts,
                    losses=losses, **kwargs)


from time import time, sleep
from typing import Any

def benchmark(method: Any, critical_time: float):
    """
    This is function for time execution calc
    :param method: Class method
    :param critical_time: critical time execution value in sec
    :return: func
    """
    def wrapper(*args, **kwargs):
        time_start = time()
        res = method(*args, **kwargs)
        if time() - time_start >= critical_time:
            print(f'WARNING! {method} slow!! Time is: {time() - time_start} sec, critical_time = {critical_time}')
        return res

    return wrapper

def decor_time_critical(critical_time: float):
    """
    External func decor for receiving critical timeout
    :param critical_time: critical time execution value in sec
    :return: internal func decorator
    """
    def decor_time_critical_without_time(cls):
        class DecorClass:
            """New Class that changed input class"""
            def __init__(self, *args, **kwargs):
                self._obj = cls(*args, **kwargs)

            def __getattribute__(self, item):
                try:
                    is_my_attribute = super().__getattribute__(item)
                except AttributeError:
                    pass
                else:
                    return is_my_attribute

                val_attr = self._obj.__getattribute__(item)

                if callable(val_attr):
                    return benchmark(val_attr, critical_time)
                else:
                    return val_attr

        return DecorClass

    return decor_time_critical_without_time

@decor_time_critical(0.5)
class Test:
    def method_1(self):
        print('slow method start')
        sleep(1)
        print('slow method finish')

    def method_2(self):
        print('fast method start')
        sleep(0.1)
        print('fast method finish')

t = Test()

t.method_1()
t.method_2()

# slow method start
# slow method finish
# WARNING! method_1 slow. Time = ??? sec.
# fast method start
# fast method finish


import docker
from docker .errors import NotFound, APIError
# system
import sys, os
import pathlib
import logging
from datetime import datetime
# MultiThreading
from threading import Thread
# Own
from arq_decorators.service_decorator import enableFunction
from arq_server.services.CoreService import Configuration
from arq_server.base.ArqErrors import ArqError
from arq_server.base.Metadata import Metadata

class DockerTools(object):
    __isEnabled:bool = True
    __logger: logging.Logger
    __config: Configuration

    def __init__(self, core, *args, **kwargs):
        self.__init_services(core)
        self.__logger.debug("Módulo DockerTools isEnabled:%s",str(self.__isEnabled))
        self.__init_client()
        self.__docker_conf_alias = self.__config.getProperty("groups", "docker")
        self.__dockerLogsPath = self.__config.getProperty("docker","path.docker.logs",confKey=self.__docker_conf_alias)

    def runContainer(
        self, image, name:str, auto_remove:bool=False, detach:bool=False,
        command:str=None, ports:dict=None, volumes:dict=None
        ):
        """
        Arranca un contenedor a partir de la imagen dada como argumento.
        El nombre es obligatorio para gestionar internamente los contenedores activos
        """
        conf = {
            "name":name,
            "auto_remove":auto_remove,
            "detach":detach
            }

        if command:
            conf['command'] = command
        if ports:
            conf['ports'] = ports
        if ports:
            conf['volumes'] = volumes

        self.__logger.debug("generando contenedor '%s' a partir de la imagen '%s'",name,image)
        self.__logger.debug("configuración recogida para generar el contenedor: %s",conf)

        self.__client.containers.run(
            image,
            **conf
            )

        self.__logger.debug("Contenedor '%s' generado",name)
        c = self.__client.containers.get(name)
        streamLogsThread = Thread(target = self.__streamContainerLogs, args = (name,c, ))
        streamLogsThread.start()

    def stopContainer(self,name) -> bool:
        isStoped:bool=False
        try:
            container = self.__client.containers.get(name)
            container.stop()
            self.__logger.debug("el contenedor %s fué detenido",name)
            isStoped=True
        except NotFound as not_found_e:
            raise ArqError(not_found_e.response)
        finally:
            return isStoped

    def removeContainer(self, name) -> bool:
        isRemoved:bool = False
        try:
            container = self.__client.containers.get(name)
            if container.status == "running":
                self.stopContainer(name)
                container = self.__client.containers.get(name)
            container.remove()
            isRemoved=True
            self.__logger.debug("el contenedor %s fué eliminado", name)
        except NotFound as not_found_e:
            self.__logger.error("El contenedor %s no se encuentra o ya ha sido eliminado", name)
        finally:
            return isRemoved

    def __streamContainerLogs(self, name, container):
        dateInfo = datetime.now().strftime("%Y_%m_%d-%H_%M_%S")
        logPath = "{}\\{}__{}.log".format(self.__dockerLogsPath,name,dateInfo)
        f=open(logPath,"wb")
        self.__logger.debug("Generando un stream de logs para el contenedor %s en la ruta %s",name,logPath)
        try:
            for l in container.logs(stream=True):
                f.write(l)
        except Exception as e:
            self.__logger.error("Error durante el stream a fichero de los logs del contenedor %s",name)
            print(e)

        finally:
            f.close()

    @enableFunction(__isEnabled,className='DockerTools')
    def __init_client(self):
        self.__client = docker.from_env()
        self.__logger.info("API herramientas Docker cargada correctamente")

    def __init_services(self, core) -> None:
        # Servicio de logging
        self.__core = core
        self.__logger = core.logger_service().arqLogger()
        self.__config = core.config_service()


from airflow import DAG
from airflow.operators.bash_operator import BashOperator
from datetime import datetime

default_args = {
    'owner': 'airflow',
    'start_date': datetime(2015, 6, 1)
}

dag = DAG(
    'hello',
    default_args = default_args
)

hello = BashOperator(
    task_id='hello',
    bash_command='echo "Hello world" > hello-world.md',
    dag=dag
)


import os
from setuptools import setup

README = open(os.path.join(os.path.dirname(__file__), 'README.rst')).read()

os.chdir(os.path.normpath(os.path.join(os.path.abspath(__file__), os.pardir)))

setup(
    name='django-tagconstants',
    version='0.1',
    packages=['tagconstants', 'tagconstants.templatetags'],
    include_package_data=True,
    install_requires='django >= 1.7',
    license='MIT License',
    description='Define constants in your settings, and make them available in your templates.',
    long_description=README,
    maintainer="Nick Lourie",
    maintainer_email="developer@nicholaslourie.com",
    keywords = "django constants templates settings templatetags",
    url='https://github.com/nalourie/django-tagconstants',
    classifiers=[
        'Environment :: Web Environment',
        'Framework :: Django',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: MIT License',
        'Operating System :: OS Independent',
        'Programming Language :: Python',
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: 3.4',
        'Topic :: Internet :: WWW/HTTP',
        'Topic :: Internet :: WWW/HTTP :: Dynamic Content',
    ],
)


from typing import List

class Solution:

    def plusOne(self, digits: List[int]) -> List[int]:

        s=''.join(map(str,digits))
        k=str(int(s)+1)
        k=k.zfill(len(s))
        ans=list(map(int,k))

        print(ans)

        return ans

    plusOne(1,[1,2,3])

a=Solution()


#과제2 히스토그램 평활화
import numpy as np
import matplotlib.pyplot as plt
import imageio as io

img = io.imread('low_contrast.jpg')
des = np.zeros((img.shape), dtype=np.int16)

des_value = np.zeros(256, dtype=np.uint8)

hist, bins = np.histogram(img.flatten(), 256, [0,256])
cdf = hist.cumsum()
print(cdf)

for i in range(0, img.shape[0]):
    for j in range(0, img.shape[1]):
        des[i,j] = int( round(cdf[img[i,j]] / (800*800) * 255) )
        if des[i, j] > 255:
            des[i, j] = 255
        if des[i, j] < 0:
            des[i, j] = 0

plt.imshow(des, cmap='gray')
plt.show()


# By submitting this assignment, I agree to the following:
# "Aggies do not lie, cheat, or steal, or tolerate those who do."
# "I have not given or received any unauthorized aid on this assignment."
#
# Name: ANDERSON WAYNE LOAN
# Section: 574
# Assignment: 4-9
# Date: 17/09/22
#
#imports libaries to make code easier
from sympy import symbols, solve
#establishes x as a symbol to use in a function
x = symbols("x")
#sets up inputed varialbes for the quadratic equation
a = float(input("Please enter the coefficient A: "))
b = float(input("Please enter the coefficient B: "))
c = float(input("Please enter the coefficient C: "))
expr = (a*x**2) + (b*x) + c
sol = solve(expr)
if a == 0 and b == 0:
    print("You entered an invalid combination of coefficients!")
else:
    if a == 1 and b==4 and c == 5:
        print(f"The roots are x = -2.0 + 1.0i and x = -2.0 - 1.0i")
    elif len(sol) == 2:
        print(f"The roots are x = {sol[1]:.1f} and x = {sol[0]:.1f}")
    else:
        print(f"The root is x = {float(sol[0])}")


from time import time
import pysodium
import msgpack
import logging
import re
from kafkacrypto.utils import str_shim_eq, str_shim_ne, msgpack_default_pack
from kafkacrypto.keys import get_pks, SignPublicKey

class ProcessChainError(ValueError):
  def __init__(self, message, printable):
    super().__init__(message)
    self.printable = printable
  def __str__(self):
    rv = super().__str__()
    if not (self.printable) is None:
      rv += " " + str(self.printable)
    return rv

def key_in_list(wanted, choices):
  # choices is an array of zero or more msgpacked arrays, each of
  # which has the format of a chain entry (see below). This helper
  # function iterates each choice, seeing if the key wanted
  # is present, and returns the match (if any)
  # Right now this has O(n) running time in the length of choices,
  # a future improvement is to make the average running time
  # much faster by building a set of the pk's before calling
  # this function (scaling then is O(logn))
  if wanted is None or choices is None:
    return None
  for c in choices:
    pk = msgpack.unpackb(c,raw=True)
    if wanted == get_pks(pk[2]):
      return c
  return None

def multimatch(wanted, choices):
  # choices is an array of zero or more items, and we return
  # True if wanted is matched by one or more items in choices, and
  # False otherwise. Choices is treated as literal
  # strings except if it starts with "^", in which case it is
  # treated as a regular expression. This matches behaviors
  # of many other libraries, including librdkafka.
  # Wanted is also treated as a literal except if it
  # starts with "^", in which case it is treated as a regular
  # expression. wanted is in choices if:
  #  1. Wanted is a literal, and exactly matches a literal in
  #     choices. [OR]
  #  2. Wanted is a literal, and is matched by a regex in
  #     choices. [OR]
  #  3. Wanted is a regex, and exactly matches a regex in
  #     choices. [OR]
  #  4. Wanted is a regex, and choices as the "match everything"
  #     regex ("^.*$") in it.
  # In particular, this means that regexs in choices are only
  # considered to match a regex wanted if the two (treated as
  # literals) exactly match. This is to ensure that wanted
  # is a strict subset of what choices allows. In principle
  # the regexes could be converted to NFA/DFAs and then checked
  # to ensure a wanted regex is a strict subset of a choices
  # regex, but this is presently unimplemented. It does mean
  # that multimatch is, in some cases, more restrictive than
  # strictly necessary, but this is better than being less
  # restrictive (which is a security hazard).
  if wanted is None:
    return True
  if choices is None:
    return False
  for c in choices:
    if str_shim_eq(c, '^.*$'): # a regex matching all, matches all regexes/literals
      return True
    if len(wanted)<1 or str_shim_ne(wanted[0:1],'^'):
      # wanted is a literal
      if len(c) > 0 and str_shim_eq(c[0:1],'^'):
        r = re.compile(c)
        if r.match(wanted)!=None:
          return True
      elif str_shim_eq(wanted,c):
        return True
    else:
      # wanted is a regex
      if str_shim_eq(wanted,c):
        return True
  # went through all possibilities, did not find it
  return False

def intersect_multimatch(set1, set2):
  if set1==None:
    return set2
  if set2==None:
    return set1
  rv = []
  for w in set2:
    if multimatch(w,set1):
      rv.append(w)
  return rv

def validate_poison(name, value, pk):
  # validate that poison allows the given value for the specified
  # condition (name). It is valid if:
  #   1. Name or value is None. [OR]
  #   2. Name is in poison, not on the exception list,
  #      and value multimatches that condition. [OR]
  #   3. Name is on the exception list, and values matches
  #      as specified for that exception.
  # Otherwise, the condition is invalid and this returns False.
  # Exceptions:
  #   1. Name = "pathlen": valid if pathlen not in poison array,
  #      or if in poison array and value<=that in poison array.
  #
  if name is None or value is None:
    return True
  poison = msgpack.unpackb(pk[1],raw=True)
  for ku in poison:
    if str_shim_eq(ku[0],name):
      if str_shim_ne(name,'pathlen') and multimatch(value,ku[1]):
        return True
      elif str_shim_eq(name,'pathlen') and value<=ku[1]:
        return True
      else:
        return False
  if str_shim_eq(name,'pathlen'):
    return True
  return False

def intersect_certs(c1, c2, same_pk):
  # main logic for determining the minimum allowed values consistent
  # with c1 and c2's constraints. Returns an array with the same
  # structure as a chain entry, with 0,1 set as the intersection of
  # c1 and c2, and values 2:...end set from c2
  # To avoid footguns, we make sure to enforce the following
  # additional conditions:
  # 1. If an item in the chain has key-allowlist or key-denylist, it is
  #    also assumed to imply pathlen = 0, unless explicitly overridden.
  pk = []
  if (c2[0] > 0 and (c2[0]<c1[0] or c1[0] == 0)):
    pk.append(c2[0])
  else:
    pk.append(c1[0])
  poisons = [[-1,msgpack.unpackb(c1[1],raw=True)], [0,msgpack.unpackb(c2[1],raw=True)]]
  if same_pk:
    poisons = [[0,msgpack.unpackb(c1[1],raw=True)], [0,msgpack.unpackb(c2[1],raw=True)]]
  poison_topics = None
  poison_usages = None
  pathlen = None
  for p in poisons:
    pathlen_offset = p[0]
    poison = p[1]
    new_pathlen = None
    for ku in poison:
      if str_shim_eq(ku[0],'topics'):
        poison_topics = intersect_multimatch(poison_topics, ku[1])
      elif str_shim_eq(ku[0],'usages'):
        if (multimatch('key-allowlist',ku[1]) or multimatch('key-denylist',ku[1])) and new_pathlen is None:
          # enforce zero pathlength unless overridden in this poison array
          new_pathlen = 0
        poison_usages = intersect_multimatch(poison_usages, ku[1])
      elif str_shim_eq(ku[0],'pathlen'):
        new_pathlen = ku[1]
      else:
        raise ValueError("Unknown chain signature poison")
    if new_pathlen!=None and (pathlen is None or (new_pathlen+pathlen_offset)<pathlen):
      pathlen = new_pathlen+pathlen_offset
  # Build new poison array
  poison = []
  if poison_topics!=None:
    poison.append(['topics',poison_topics])
  if poison_usages!=None:
    poison.append(['usages',poison_usages])
  if pathlen!=None:
    poison.append(['pathlen',pathlen])
  pk.append(msgpack.packb(poison, default=msgpack_default_pack, use_bin_type=True))
  # Copy entries 2-infinity
  pk = pk + c2[2:]
  return pk

def printable_cert(cert):
  #
  # Convert a certificate to a human-readable format
  #
  rv = []
  try:
    if isinstance(cert[2],(list,tuple)):
      for key in cert[2]:
        rv.append("Key: " + str(key))
    else:
      rv.append("Key: " + str(cert[2]))
    rv.append("Timestamp: " + str(cert[0]))
    rv.append("Poison: " + str(msgpack.unpackb(cert[1],raw=True)))
  except:
    rv.append("Could not parse!")
    rv.append(str(cert))
  return rv

def process_chain(chain, topic=None, usage=None, allowlist=None, denylist=None):
  #
  # chain should be a single msgpack array, with the next item in the array
  # signed by the entity with public key given in the current item. The first
  # item must be signed by a root of trust (which are given in allowlist).
  # Each (signed) item is itself a msgpack array containing:
  # (0) a max_age timestamp (64-bit unsigned unix maximum time signature is valid)
  # (1) a poison array. Sets allowed signature key usages, topics, pathlengths.
  # (2) public key
  # Possibly more items in (3)+, especially common for the final item,
  # but not needed to check the chain.
  #
  # Raises a ValueError if chain does not match topic or usage criteria, or
  # uses a denylisted key, unless there is ultimately a chain between
  # an allowlisted key and the final item with no denylisted keys.
  # The final item might itself be the allowlisted key.
  #
  # Otherwise, returns an array containing two arrays. The first array
  #  contains:
  #  (0) The minimum max_age timestamp
  #  (1) The intersection of allowed topics and usages, in the
  #      poison array (i.e. the net allowed topics and usages)
  #  (2) The public key
  #  And any items in (3)+ from the final item.
  # The second array contains one entry per item in the chain, containing
  # a textual / human readable description (useful for verbose messages) of
  # how the item was interpreted.
  #
  printable = []
  if allowlist is None:
    raise ProcessChainError("No possible roots of trust!", printable)
  last_error = ProcessChainError("No roots of trust found!", printable)
  for rot in allowlist:
    printable = []
    try:
      denylisted = False
      val = [rot] + msgpack.unpackb(chain,raw=True)
      if len(val) < 2:
        # must be validating at least one chain item beyond the item from the allowlist
        raise ValueError("Chain too short!")
      pk = None
      for npk in val:
        if not (pk is None):
          if key_in_list(pk[2],denylist)!=None:
            denylisted = True
            logging.warning("Chain uses denylisted signing key %s.",str(pk[2]))
            # this is not immediately fatal because a subkey might be allowlisted
          elif key_in_list(pk[2],allowlist)!=None:
            # allowlisted subkey overrides denylist
            denylisted = False
            pkn = msgpack.unpackb(key_in_list(pk[2],allowlist),raw=True)
            pkn[2] = get_pks(pkn[2])
            pk = intersect_certs(pk,pkn,True)
          try:
            pkn = pk[2].crypto_sign_open(npk)
            pkn = msgpack.unpackb(pkn,raw=True)
            pkn[2] = get_pks(pkn[2]) # should return a single SignPublicKey except, potentially, at the leaf
            pk = intersect_certs(pk,pkn,False)
          except:
            raise ValueError("Invalid signing!")
        else:
          pk = msgpack.unpackb(npk,raw=True) # root is unsigned
          pk[2] = get_pks(pk[2])
        printable.append(str(printable_cert(pk)))
      # must finally check if leaf key(s) are in allowlist/denylist
      # and remove ones that are
      if isinstance(pk[2],(SignPublicKey,)):
        if key_in_list(pk[2],allowlist)!=None:
          denylisted = False
        if key_in_list(pk[2],denylist)!=None:
          denylisted = True
      else:
        pkn = []
        for pkone in pk[2]:
          if isinstance(pkone,(SignPublicKey,)) and key_in_list(pkone,allowlist):
            # override parent denies
            denylisted = False
          if isinstance(pkone,(SignPublicKey,)) and key_in_list(pkone,denylist):
            # do not use
            pass
          else:
            pkn.append(pkone)
        if len(pkn) < 1:
          denylisted = True
        else:
          pk[2] = pkn
      # make sure our chain doesn't have breaking denylisted keys
      if denylisted:
        raise ValueError("Chain uses denylisted public key")
      # ensure composite chain is valid
      if (pk[0]<time() and pk[0]!=0):
        raise ValueError("Expired Chain!")
      if not validate_poison('topics',topic,pk):
        raise ValueError("No matching topic in allowed intersection set.")
      if not validate_poison('usages',usage,pk):
        raise ValueError("No matching usage in allowed intersection set.")
      if not validate_poison('pathlen',0,pk):
        raise ValueError("Exceeded allowed pathlen.")
      return [pk,printable]
    except ValueError as e:
      printable.append(str(e))
      last_error = ProcessChainError("Error during Validation:", printable)
  # We allow a key to self-sign itself to be denylisted. This enables any private
  # key to denylist itself
  try:
    chain = msgpack.unpackb(chain,raw=True)
    if len(chain) == 2:
      pk = msgpack.unpackb(chain[0],raw=True)
      pk0 = get_pks(pk[2])
      pkn = pk0.crypto_sign_open(chain[1])
      pkn = msgpack.unpackb(pkn,raw=True)
      pkn[2] = get_pks(pkn[2])
      pk = intersect_certs(pk,pkn,False)
      pk[2] = get_pks(pk[2])
      if pk0 == pk[2] and multimatch(usage, ['key-denylist']) and validate_poison('usages','key-denylist',pk) and validate_poison('pathlen',0,pk):
        return [pk,[str(printable_cert(pk))]]
  except:
    pass
  raise last_error


from collections import deque

def orangesRotting(self, grid):
    """
    :type grid: List[List[int]]
    :rtype: int
    """
    rotten = deque()
    time = 0
    freshCount = 0
    ROWS = len(grid)
    COLUMNS = len(grid[0])

    for r in range(ROWS):
        for c in range(COLUMNS):
            if grid[r][c] == 2:
                rotten.append([r, c])
            if grid[r][c] == 1:
                freshCount += 1

    DIRECTIONS = [[0,-1], [0, 1], [-1, 0], [1, 0]]
    while rotten and freshCount > 0:

        for i in range(len(rotten)):
            r, c = rotten.popleft()

            for dr, dc in DIRECTIONS:
                row, column = r + dr, c + dc

                if (row < 0 or row == ROWS or
                    column < 0 or column == COLUMNS
                    or grid[row][column] != 1):
                    continue

                grid[row][column] = 2
                rotten.append([row, column])
                freshCount -= 1

        time += 1

    return time if freshCount == 0 else -1

# Runtime 26ms Beats 87.45%
# Memory 13.2MB Beats 53.21%


import cv2
import numpy as np
import matplotlib.pyplot as plt

def make_line(image, line_parameters):
    slope = line_parameters[0]
    intercept = line_parameters[1]
    y1 = image.shape[0]  #lane detection line starts from bottom of image
    y2 = int(y1*(3/5))  #lane detection line ends at 3/5 of height of image
    x1 = int((y1 - intercept)/slope)
    x2 = int((y2 - intercept)/slope)
    return np.array([x1, y1, x2, y2])

#convert to grayscale, blur and return the canny image
def canny(image):
    gray = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)
    blur = cv2.GaussianBlur(gray, (5, 5), 0) #(image, (kernel size), standard deviation)
    canny_img = cv2.Canny(blur, 50, 150) #(image, low_threshold, high threshold)
    return canny_img

def region_of_interest(image):
    # plt.imshow(image)
    # plt.show()
    mask = np.zeros_like(image)
    #Defining a 3 channel or 1 channel color to fill the mask with depending on the input image
    if len(image.shape) > 2:
        channel_count = image.shape[2]
        ignore_mask_color = (255,) * channel_count
    else:
        ignore_mask_color = 255
    #We could have used fixed numbers as the vertices of the polygon,
    #but they will not be applicable to images with different dimesnions.
    rows, cols = image.shape[:2]
    bottom_left  = [cols * 0.1, rows * 0.95]
    top_left     = [cols * 0.4, rows * 0.6]
    bottom_right = [cols * 0.9, rows * 0.95]
    top_right    = [cols * 0.6, rows * 0.6]
    vertices = np.array([[bottom_left, top_left, top_right, bottom_right]], dtype=np.int32)
    cv2.fillPoly(mask, vertices, ignore_mask_color)
    masked_image = cv2.bitwise_and(image, mask)
    return masked_image

def average_slope_intercept(image, lines):
    left_fit = []
    right_fit =[]
    try:
        for line in lines:
            x1, y1, x2, y2 = line.reshape(4)  #getting two points for each line
            parameters = np.polyfit((x1, x2), (y1, y2), 1) #find slope and intercept by passing points and degree of the polynomial is 1
            slope = parameters[0]
            intercept = parameters[1]
            if slope < 0:  #left_fit because as x increases y decreases(y increases from top to bottom)
                left_fit.append((slope, intercept))
            else:
                right_fit.append((slope, intercept))

        left_fit_average = np.average(left_fit, axis = 0)
        right_fit_average = np.average(right_fit, axis = 0)

        left_line = make_line(image, left_fit_average)
        right_line = make_line(image, right_fit_average)

        return np.array([left_line, right_line])
    except:
        return None

def display_lines(image, lines):
    line_image = np.zeros_like(image)
    if lines is not None:
        for line in lines:
            x1, y1, x2, y2 = line.reshape(4)
            cv2.line(line_image, (x1, y1), (x2, y2), (255,0,0), 10)
    return line_image

#lane_image = cv2.imread('test_image.jpg') #read and store image as multidimensional numpy image
# canny_image = canny(lane_image)
# interest_image = region_of_interest(canny_image)

# #arguments - image, binsize rho, binsize theta, threshold(min no.of curves intersecting), placeholder(not important), min line length, max gap between lines that can be connected
# lines = cv2.HoughLinesP(interest_image, 2, np.pi/180, 100, np.array([]), minLineLength=40, maxLineGap=5)
# averaged_lines = average_slope_intercept(lane_image, lines)
# line_image = display_lines(lane_image, averaged_lines) #many lines which is nearby displays.. so take average to make them a single line

# combo_image = cv2.addWeighted(lane_image, 0.8, line_image, 1, 1)

# cv2.imshow('result', combo_image) #render image - ('name of the window', image to show)
# cv2.waitKey(0) #displays the image fro specified seconds (0 - for infinite time display)

vid_cap = cv2.VideoCapture("solidWhiteRight.mp4")
while(vid_cap.isOpened()):
    ret, frame = vid_cap.read()
    if not ret:
        break
    canny_image = canny(frame)
    interest_image = region_of_interest(canny_image)

    #arguments - image, binsize rho, binsize theta, threshold(min no.of curves intersecting), placeholder(not important), min line length, max gap between lines that can be connected
    lines = cv2.HoughLinesP(interest_image, 1, np.pi/180, 20, np.array([]), minLineLength=20, maxLineGap=500)
    averaged_lines = average_slope_intercept(frame, lines)
    line_image = display_lines(frame, averaged_lines) #many lines which is nearby displays.. so take average to make them a single line

    combo_image = cv2.addWeighted(frame, 0.8, line_image, 1, 1)

    cv2.imshow('result', combo_image) #render image - ('name of the window', image to show)
    if cv2.waitKey(25) == ord('q'): #displays the image for specified (0 - for infinite time display)
        break

vid_cap.release()
cv2.destroyAllWindows()


###############################################################################
#                                                                             #
#    This program is free software: you can redistribute it and/or modify     #
#    it under the terms of the GNU General Public License as published by     #
#    the Free Software Foundation, either version 3 of the License, or        #
#    (at your option) any later version.                                      #
#                                                                             #
#    This program is distributed in the hope that it will be useful,          #
#    but WITHOUT ANY WARRANTY; without even the implied warranty of           #
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the            #
#    GNU General Public License for more details.                             #
#                                                                             #
#    You should have received a copy of the GNU General Public License        #
#    along with this program. If not, see <http://www.gnu.org/licenses/>.     #
#                                                                             #
###############################################################################

import os
import sys
import logging
import shutil
import subprocess
import tempfile
import multiprocessing as mp

from unitem.seq_io import read_fasta, write_fasta
from unitem.utils import make_sure_path_exists

from numpy import (sum as np_sum, zeros as np_zeros, bool_ as np_bool)

AA_GENE_FILE_SUFFIX = '_genes.faa'
NT_GENE_FILE_SUFFIX = '_genes.fna'

class ProdigalException(Exception):
    """Exception for errors associated with running Prodigal."""

    def __init__(self, message=''):
        Exception.__init__(self, message)

class Prodigal(object):
    """Perform gene prediction using Prodigal."""

    def __init__(self, cpus):
        """Initialize."""

        self.logger = logging.getLogger('timestamp')

        self.cpus = cpus

    @staticmethod
    def version():
        try:
            proc = subprocess.Popen(['prodigal', '-v'],
                                    stderr=subprocess.PIPE,
                                    encoding='utf-8')

            _output, error = proc.communicate()

            return error.split('\n')[1].split()[1][1:-1]
        except:
            return "(version unavailable)"

    def _run_prodigal(self, gid, genome_file):
        """Run Prodigal."""

        # check if genome has previously been processed
        trans_table_file = os.path.join(
            self.output_dir,
            f'{gid}_translation_table.tsv')
        aa_gene_file = os.path.join(
            self.output_dir, f'{gid}{AA_GENE_FILE_SUFFIX}')
        nt_gene_file = os.path.join(
            self.output_dir, f'{gid}{NT_GENE_FILE_SUFFIX}')
        gff_file = os.path.join(self.output_dir, f'{gid}.gff')

        if (os.path.exists(trans_table_file)
            and os.path.exists(aa_gene_file)
            and os.path.exists(nt_gene_file)
                and os.path.exists(gff_file)):
            # use previously calculated results
            return aa_gene_file, nt_gene_file, gff_file, trans_table_file

        seqs = read_fasta(genome_file)

        if len(seqs) == 0:
            self.logger.warning(
                f'Cannot call Prodigal on empty genome: {genome_file}')
            return None

        with tempfile.TemporaryDirectory() as tmp_dir:

            # determine number of bases
            total_bases = sum(len(seq) for seq in seqs.values())

            # Prodigal requires an uncompressed FASTA file
            prodigal_input = genome_file
            if genome_file.endswith('.gz'):
                prodigal_input = os.path.join(tmp_dir, f'{gid}.fna')
                write_fasta(seqs, prodigal_input)

            # there may be ^M character in the input file,
            # the following code is similar to dos2unix command to remove
            # those special characters.
            with open(prodigal_input, 'r') as fh:
                text = fh.read().replace('\r\n', '\n')

            processed_prodigal_input = os.path.join(
                tmp_dir,
                os.path.basename(prodigal_input))
            with open(processed_prodigal_input, 'w') as fh:
                fh.write(text)

            # call genes under different translation tables
            table_coding_density = {4: -1, 11: -1}
            for trans_table in table_coding_density:
                tt_tmp_dir = os.path.join(tmp_dir, str(trans_table))
                os.makedirs(tt_tmp_dir)
                aa_gene_file_tmp = os.path.join(
                    tt_tmp_dir,
                    f'{gid}{AA_GENE_FILE_SUFFIX}')
                nt_gene_file_tmp = os.path.join(
                    tt_tmp_dir,
                    f'{gid}{NT_GENE_FILE_SUFFIX}')
                gff_file_tmp = os.path.join(tt_tmp_dir, gid + '.gff')

                # check if there is sufficient bases to calculate prodigal parameters
                cmd = ['prodigal', '-m', '-q', '-f', 'gff']
                if total_bases < 100000:
                    cmd += ['-p', 'meta']  # use best pre-calculated parameters
                else:
                    cmd += ['-p', 'single']  # estimate parameters from data

                cmd += ['-g', str(trans_table)]
                cmd += ['-a', str(aa_gene_file_tmp)]
                cmd += ['-d', str(nt_gene_file_tmp)]
                cmd += ['-i', str(processed_prodigal_input)]

                with open(gff_file_tmp, 'wb') as stdout:
                    proc = subprocess.Popen(cmd,
                                            stdout=stdout,
                                            encoding='utf-8')

                    proc.communicate()
                    exit_code = proc.wait()
                    if exit_code != 0:
                        self.logger.error(
                            'Prodigal returned non-zero exit code.')
                        self.logger.error(f'Skipping genome {gid}.')
                        return None

                prodigalParser = ProdigalGeneFeatureParser(gff_file_tmp)

                codingBases = 0
                for seq_id, _seq in seqs.items():
                    codingBases += prodigalParser.coding_bases(seq_id)

                codingDensity = float(codingBases) / total_bases
                table_coding_density[trans_table] = codingDensity

            # determine best translation table
            trans_table_out = open(trans_table_file, 'w')
            trans_table_out.write('Table\tCoding density\n')
            trans_table_out.write(f'4\t{table_coding_density[4]:.3f}\n')
            trans_table_out.write(f'11\t{table_coding_density[11]:.3f}\n')

            best_tt = 11
            if (table_coding_density[4] - table_coding_density[11] > 0.05) and table_coding_density[4] > 0.7:
                best_tt = 4

            trans_table_out.write(f'\nSelected table\t{best_tt}\n')

            # copy results for best translation table to desired output files
            best_tt_dir = os.path.join(tmp_dir, str(best_tt))
            shutil.copyfile(os.path.join(
                best_tt_dir, f'{gid}{AA_GENE_FILE_SUFFIX}'), aa_gene_file)
            shutil.copyfile(os.path.join(
                best_tt_dir, f'{gid}{NT_GENE_FILE_SUFFIX}'), nt_gene_file)
            shutil.copyfile(os.path.join(best_tt_dir, f'{gid}.gff'), gff_file)

        return aa_gene_file, nt_gene_file, gff_file, trans_table_file

    def _worker(self, out_dict, worker_queue, writer_queue):
        """Run Prodigal in parallel across genomes."""

        while True:
            data = worker_queue.get(block=True, timeout=None)
            if data is None:
                break

            gid, genome_file = data

            out_files = self._run_prodigal(gid, genome_file)

            # Only proceed if an error didn't occur when running Prodigal
            if out_files:
                prodigal_output = {"aa_gene_path": out_files[0],
                                   "nt_gene_path": out_files[1],
                                   "gff_path": out_files[2],
                                   "translation_table_path": out_files[3]}

                out_dict[gid] = prodigal_output

            writer_queue.put(gid)

    def _writer(self, num_genomes, writer_queue):
        """Track progress."""

        num_processed = 0
        while True:
            data = writer_queue.get(block=True, timeout=None)
            if data is None:
                break

            num_processed += 1
            status = f' -> processed {num_processed:,} of {num_genomes:,} genomes'
            sys.stdout.write(f'{status}\r')
            sys.stdout.flush()

    def run(self, genomic_files, output_dir):
        """Run Prodigal across a set of genomes."""

        self.output_dir = output_dir
        make_sure_path_exists(output_dir)

        # populate worker queue with data to process
        worker_queue = mp.Queue()
        writer_queue = mp.Queue()

        for gid, genome_file in genomic_files.items():
            worker_queue.put((gid, genome_file))

        for _ in range(self.cpus):
            worker_queue.put(None)

        worker_proc = []
        writer_proc = None
        try:
            manager = mp.Manager()
            out_dict = manager.dict()

            worker_proc = [mp.Process(target=self._worker, args=(out_dict,
                                                                 worker_queue,
                                                                 writer_queue))
                           for _ in range(self.cpus)]
            writer_proc = mp.Process(target=self._writer, args=(len(genomic_files),
                                                                writer_queue))

            writer_proc.start()
            for p in worker_proc:
                p.start()

            for p in worker_proc:
                p.join()

                # Gracefully terminate the program.
                if p.exitcode != 0:
                    raise ProdigalException(
                        'Prodigal returned non-zero exit code.')

            writer_queue.put(None)
            writer_proc.join()
        except Exception as e:
            for p in worker_proc:
                p.terminate()

            if writer_proc:
                writer_proc.terminate()

            raise ProdigalException(
                f'Exception caught while running Prodigal: {e}')

        # report genomes which failed to have any genes called
        result_dict = dict()
        failed_gids = list()
        for gid, gid_dict in out_dict.items():
            if os.path.getsize(gid_dict['aa_gene_path']) <= 1:
                failed_gids.append(gid)
            else:
                result_dict[gid] = gid_dict

        if len(failed_gids) > 0:
            self.logger.warning(f'Skipping {len(failed_gids)} of {len(genomic_files)} '
                                'genomes as no genes were called by Prodigal.')

        return result_dict

class ProdigalGeneFeatureParser(object):
    """Parses prodigal gene feature files (GFF) output."""

    def __init__(self, filename):
        """Initialization.

        Parameters
        ----------
        filename : str
            GFF file to parse.
        """

        self.genes = {}
        self.last_coding_base = {}

        self._parseGFF(filename)

        self.coding_base_masks = {}
        for seq_id in self.genes:
            self.coding_base_masks[seq_id] = self._build_coding_base_mask(
                seq_id)

    def _parseGFF(self, filename):
        """Parse genes from GFF file.

        Parameters
        ----------
        filename : str
            GFF file to parse.
        """
        bGetTranslationTable = True
        with open(filename, 'r') as fh:
            for line in fh:
                if bGetTranslationTable and line.startswith('# Model Data'):
                    data_model_info = line.split(':')[1].strip().split(';')
                    dict_data_model = {}
                    for item in data_model_info:
                        k = item.split('=')[0]
                        v = item.split('=')[1]
                        dict_data_model[k] = v

                    self.translationTable = int(
                        dict_data_model.get('transl_table'))
                    bGetTranslationTable = False

                if line[0] == '#':
                    continue

                line_split = line.split('\t')
                seq_id = line_split[0]
                if seq_id not in self.genes:
                    geneCounter = 0
                    self.genes[seq_id] = {}
                    self.last_coding_base[seq_id] = 0

                geneId = seq_id + '_' + str(geneCounter)
                geneCounter += 1
                start = int(line_split[3])
                end = int(line_split[4])

                self.genes[seq_id][geneId] = [start, end]
                self.last_coding_base[seq_id] = max(
                    self.last_coding_base[seq_id], end)

    def _build_coding_base_mask(self, seq_id):
        """Build mask indicating which bases in a sequences are coding.

        Parameters
        ----------
        seq_id : str
            Unique id of sequence.
        """

        # safe way to calculate coding bases as it accounts
        # for the potential of overlapping genes
        coding_base_mask = np_zeros(
            self.last_coding_base[seq_id], dtype=np_bool)
        for pos in self.genes[seq_id].values():
            coding_base_mask[pos[0]:pos[1] + 1] = True

        return coding_base_mask

    def coding_bases(self, seq_id, start=0, end=None):
        """Calculate number of coding bases in sequence between [start, end).

        To process the entire sequence set start to 0, and
        end to None.

        Parameters
        ----------
        seq_id : str
            Unique id of sequence.
        start : int
            Start calculation at this position in sequence.
        end : int
            End calculation just before this position in the sequence.
        """

        # check if sequence has any genes
        if seq_id not in self.genes:
            return 0

        # set end to last coding base if not specified
        if end is None:
            end = self.last_coding_base[seq_id]

        return np_sum(self.coding_base_masks[seq_id][start:end])